Merge pull request #324 from FAIRDataPipeline/Dev

fix styling

.flake8

@@ -1,7 +1,16 @@
 [flake8]
 ignore =  E501, W503
-max-line-length = 79
+max-line-length = 88
+extend-ignore = E203
 max-complexity = 18
 select = B,C,E,F,W,T4,B9,N
 exclude =
-**/migrations/**
+    **/migrations/**
+    __pycache__
+    manage.py
+    settings.py
+    env
+    .env
+    venv
+    .venv
+    tests/data

.isort.cfg

@@ -6,3 +6,5 @@ force_grid_wrap = 0
 use_parentheses = True
 ensure_newline_before_comments = True
 line_length = 79
+max-line-length = 88
+extend-ignore = E203

fair/registry/requests.py

@@ -89,24 +89,13 @@ def _access(
     method: str = None,
     token: str = None,
     obj_path: str = None,
-    response_codes: typing.List[int] = None,
-    headers: typing.Dict[str, typing.Any] = None,
-    params: typing.Dict = None,
-    data: typing.Dict = None,
+    response_codes: typing.List[int] = [201, 200],
+    headers: typing.Dict[str, typing.Any] = {},
+    params: typing.Dict = {},
+    data: typing.Dict = {},
     files: typing.Dict = None,
     trailing_slash=True,
 ):
-    if response_codes is None:
-        response_codes = [201, 200]
-    if not headers:
-        headers: typing.Dict[str, str] = {}
-
-    if not params:
-        params: typing.Dict[str, str] = {}
-
-    if not data:
-        data: typing.Dict[str, str] = {}
-
     # Make sure we have the right number of '/' in the components
     _uri = uri
     _uri = fdp_util.check_trailing_slash(_uri)
@@ -159,7 +148,8 @@ def _access(
 
     if _request.status_code == 403:
         raise fdp_exc.RegistryAPICallError(
-            f"Failed to run method '{method}' for url {_url}, request forbidden",
+            f"Failed to run method '{method}' for url {_url}, \
+                request forbidden",
             error_code=403,
         )
     elif _request.status_code == 409:
@@ -227,7 +217,10 @@ def post(
 
     for param, value in data.copy().items():
         if not value:
-            logger.debug(f"Key in post data '{param}' has no value so will be ignored")
+            logger.debug(
+                f"Key in post data '{param}' \
+                         has no value so will be ignored"
+            )
             del data[param]
 
     return _access(
@@ -289,7 +282,9 @@ def get(
         returned data for the given request
     """
     logger.debug(
-        "Retrieving object of type '%s' from registry at '%s' with parameters: %s",
+        "Retrieving object of type '%s' \
+            from registry at '%s' \
+            with parameters: %s",
         obj_path,
         uri,
         params,
@@ -303,7 +298,8 @@ def get(
     for param, value in params.copy().items():
         if not value:
             logger.warning(
-                f"Key in get parameters '{param}' has no value so will be ignored"
+                f"Key in get parameters '{param}' \
+                    has no value so will be ignored"
             )
             del params[param]
 
@@ -367,15 +363,15 @@ def post_else_get(
 
 
 def post_upload_url(remote_uri: str, remote_token: str, file_hash: str) -> str:
-    """Function to get a tempory url to upload and object to
+    """Function to get a temporary url to upload and object to
 
     Args:
         remote_uri (str): Remote registry URL
         remote_token (str): Remote token
         file_hash (str): Hash of the file to be uploaded
 
     Returns:
-        str: A tempory url to upload the object to
+        str: A temporary url to upload the object to
     """
     _url = urllib.parse.urljoin(remote_uri, "data/")
     _url = urllib.parse.urljoin(_url, file_hash)
@@ -480,7 +476,8 @@ def put_file(upload_url: str, file_loc: str) -> bool:
         file_loc (str): Location of the file to be uploaded
 
     Raises:
-        fdp_exc.RegistryError: If the upload fails a RegistryError will be raised
+        fdp_exc.RegistryError:
+            If the upload fails a RegistryError will be raised
 
     Returns:
         bool: Will return True if the upload succeeded.
@@ -489,7 +486,8 @@ def put_file(upload_url: str, file_loc: str) -> bool:
     _req = s.put(upload_url, data=open(file_loc, mode="rb").read())
     if _req.status_code not in [200, 201]:
         raise fdp_exc.RegistryError(
-            f"File: {file_loc} could not be uploaded, Registry Returned: {_req.status_code}"
+            f"File: {file_loc} could not be uploaded,\
+            Registry Returned: {_req.status_code}"
         )
     return True
 
@@ -543,7 +541,8 @@ def download_file(url: str, chunk_size: int = 8192) -> str:
 def get_dependency_listing(
     uri: str, token: str, read_only: bool = False
 ) -> typing.Dict:
-    """Get complete listing of all objects and their registry based dependencies
+    """Get complete listing of all objects and
+    their registry based dependencies
 
     Parameters
     ----------
@@ -648,7 +647,7 @@ def get_author_exists(registry_uri, token=None, name=None, identifier=None) -> s
 
 
 def get_auth_provider(registry_uri: str, token: str = None) -> str:
-    """Retrived the auth provider from the remote registry
+    """Retrieve the auth provider from the remote registry
 
     Parameters
     ----------
@@ -667,14 +666,14 @@ def get_auth_provider(registry_uri: str, token: str = None) -> str:
     if not _response["auth_provider"]:
         raise fdp_exc.RegistryError(
             f"The remote registry {registry_uri} \
-                                    did not provide an authentication provider \
-                                    is the remote registry correcly configured?"
+            did not provide an authentication provider \
+            is the remote registry correctly configured?"
         )
     return _response["auth_provider"]
 
 
 def get_auth_url(registry_uri: str, token: str = None) -> str:
-    """Retrived the auth url from the remote registry
+    """Retrieve the auth url from the remote registry
 
     Parameters
     ----------
@@ -693,7 +692,7 @@ def get_auth_url(registry_uri: str, token: str = None) -> str:
     if not _response["auth_url"]:
         raise fdp_exc.RegistryError(
             f"The remote registry {registry_uri} \
-                                    did not provide an authentication provider \
-                                    is the remote registry correcly configured?"
+            did not provide an authentication provider \
+            is the remote registry correctly configured?"
         )
     return _response["auth_url"]

fair/registry/storage.py

@@ -59,7 +59,8 @@ def get_write_storage(uri: str, write_data_store: str, token: str) -> str:
     Raises
     ------
     fdp_exc.UserConfigError
-        If 'write_data_store' not present in the working config or global config
+        If 'write_data_store' not present
+            in the working config or global config
     """
     logger.debug("Constructing a storage root for '%s'", write_data_store)
 
@@ -256,7 +257,8 @@ def store_working_config(repo_dir: str, uri: str, work_cfg_yml: str, token: str)
             raise e
         else:
             raise fdp_exc.RegistryAPICallError(
-                f"Cannot post storage_location '{_rel_path}' with hash '{_hash}', object already exists",
+                f"Cannot post storage_location '{_rel_path}' \
+                    with hash '{_hash}', object already exists",
                 error_code=409,
             ) from e
 
@@ -618,7 +620,8 @@ def _get_url_from_data_product(
         _name = data["data_product"]
     else:
         raise fdp_exc.UserConfigError(
-            f"Failed to determine type while storing item '{label}'" "into registry"
+            f"Failed to determine type while storing item '{label}' \
+                into registry"
         )
 
     if "data_product" in data["use"]:
@@ -640,7 +643,8 @@ def _get_url_from_data_product(
             raise e
         else:
             raise fdp_exc.RegistryAPICallError(
-                f"Cannot post data_product '{_name}', duplicate already exists",
+                f"Cannot post data_product '{_name}', \
+                    duplicate already exists",
                 error_code=409,
             ) from e
 
@@ -666,7 +670,7 @@ def _get_identifier_from_data(data: typing.Dict, label: str) -> typing.Dict[str,
             _identifier["alternate_identifier"] = data["unique_name"]
         except KeyError as e:
             raise fdp_exc.UserConfigError(
-                "No identifier/alternate_identifier given for " f"item '{label}'",
+                f"No identifier/alternate_identifier given for item '{label}'",
                 hint="You must provide either a URL 'identifier', or "
                 "'unique_name' and 'source_name' keys",
             ) from e
@@ -855,10 +859,12 @@ def upload_remote_file(
     Args:
         file_loc (str): Location of the file
         remote_uri (_type_, optional): Remote Registry URL. Defaults to None.
-        remote_token (_type_, optional): Remote Registry Token. Defaults to None.
+        remote_token (_type_, optional):
+            Remote Registry Token. Defaults to None.
 
     Raises:
-        fdp_exc.FileNotFoundError: If the file does not exist a FileNotFound error will be raised.
+        fdp_exc.FileNotFoundError:
+            If the file does not exist a FileNotFound error will be raised.
     """
     if not os.path.exists(file_loc):
         raise fdp_exc.FileNotFoundError(f"File: {file_loc} does not exist")

fair/registry/sync.py

@@ -216,7 +216,8 @@ def sync_dependency_chain(
 
         if not fdp_util.is_api_url(dest_uri, _new_url):
             raise fdp_exc.InternalError(
-                f"Expected new URL '{_new_url}' to be compatible with destination URI '{dest_uri}'"
+                f"Expected new URL '{_new_url}' to be compatible with \
+                    destination URI '{dest_uri}'"
             )
 
         # Fill the new URLs dictionary with the result
@@ -237,7 +238,8 @@ def _get_new_url(
     local_data_store=None,
     public=False,
 ) -> typing.Tuple[typing.Dict, typing.List]:
-    """Internal Function to return a resgistry entry from the remote registry given an origin entry URL
+    """Internal Function to return a registry entry from the remote registry
+    given an origin entry URL
     If the entry does not exist it will be created
 
     Args:
@@ -246,15 +248,20 @@ def _get_new_url(
         dest_uri (str): Url of the destination registry
         dest_token (str): Token of the destination registry
         object_url (str): Url of the destination registry entry
-        new_urls (typing.Dict): If the entry contains registry URLS, a list of remote registry URLs for these
-        writable_data (typing.Dict): Dictionary of writable fields for the given entity type
+        new_urls (typing.Dict): If the entry contains registry URLS,
+            a list of remote registry URLs for these
+        writable_data (typing.Dict): Dictionary of writable fields for
+            the given entity type
 
     Raises:
-        fdp_exc.RegistryError: If the given new_urls are not valid for the entry a RegistryError will be raised
-        fdp_exc.InternalError: If the origin url and the destination URLs are the same an InternalError with be raised
+        fdp_exc.RegistryError: If the given new_urls are not valid for
+            the entry a RegistryError will be raised
+        fdp_exc.InternalError: If the origin url and the destination URLs
+            are the same an InternalError with be raised
 
     Returns:
-        typing.Tuple[typing.Dict, typing.List]: A tuple containing the destination registry entity
+        typing.Tuple[typing.Dict, typing.List]: A tuple containing the
+            destination registry entity
     """
     _new_obj_data: typing.Dict[str, typing.Any] = {}
     _url_fields: typing.List[str] = []
@@ -304,10 +311,11 @@ def _get_new_url(
     # If Public is true then any files will be uploaded to the remote registry
     # If local_data_store is set we're pulling to a local registry
     if public and not local_data_store:
-        # The remote data_store storage_root URL should always be the 1st storage_root
+        # The remote data_store storage_root URL should
+        # always be the 1st storage_root
         _remote_storage_root_url = urllib.parse.urljoin(dest_uri, "storage_root/1/")
-        # If the current objects a storage_root and the root is the first (data_store)
-        # then simply return the remote
+        # If the current objects a storage_root and the root
+        # is the first (data_store) then simply return the remote
         if _obj_type == "storage_root" and _obj_id == "1":
             return _remote_storage_root_url
         # If the current object is a storage_location
@@ -376,12 +384,12 @@ def sync_author(
     )
     if not new_urls:
         raise fdp_exc.RegistryError(
-            f"Auther {identifier}, could not be pushed to {dest_uri}"
+            f"Author {identifier}, could not be pushed to {dest_uri}"
         )
     new_author_url = new_urls[current_author_url]
     if not new_author_url:
         raise fdp_exc.RegistryError(
-            f"Auther {identifier}, was not be pushed to {dest_uri}"
+            f"Author {identifier}, was not be pushed to {dest_uri}"
         )
     return new_author_url
 

fair/run.py

@@ -64,7 +64,8 @@ def get_job_hash(job_dir: str) -> str:
     """
     if not os.path.exists(job_dir):
         raise fdp_exc.FileNotFoundError(
-            "Failed to find hash for job, " f"directory '{job_dir}' does not exist."
+            f"Failed to find hash for job,\
+            directory '{job_dir}' does not exist."
         )
     _directory = os.path.abspath(job_dir)
     return hashlib.sha1(_directory.encode("utf-8")).hexdigest()

fair/staging.py

@@ -237,13 +237,8 @@ def _get_code_run_entries(self, local_uri: str, job_dir: str) -> typing.List[str
         _code_run_urls = []
 
         if (
-            os.path.exists
-            (
-                _code_run_file
-            ) and open(
-                _code_run_file,
-                encoding="utf-8"
-            ).read().strip()
+            os.path.exists(_code_run_file)
+            and open(_code_run_file, encoding="utf-8").read().strip()
         ):
             self._logger.debug("Found coderuns file, extracting runs")
             _runs = [

tests/conftest.py

@@ -17,7 +17,6 @@
 from pathlib import Path
 
 import boto3
-import os
 from moto.server import ThreadedMotoServer
 
 import requests
@@ -150,7 +149,7 @@ def pyDataPipeline(tmp_path_factory):
 @pytest.fixture()
 def pySimpleModel(tmp_path):
     _repo_path = tmp_path.__str__()
-    _repo = git.Repo.clone_from(PYTHON_MODEL_GIT, _repo_path)
+    _ = git.Repo.clone_from(PYTHON_MODEL_GIT, _repo_path)
     yield _repo_path
 
 

tests/test_configuration.py

@@ -147,7 +147,7 @@ def test_user_info(mocker: pytest_mock.MockerFixture):
     if not conf.test_can_be_run(f'{fdp_id.QUERY_URLS["orcid"]}{ORCID_ID}'):
         warnings.warn("Orcid API Unavailable")
         pytest.skip("Cannot Reach Orcid API")
-    _data = fdp_id.check_github("FAIRDataPipeline")
+    _ = fdp_id.check_github("FAIRDataPipeline")
     _namepaces = {"input": "ispace", "output": "jbloggs"}
     _email = "[email protected]"
     _github_username = "FAIRDataPipeline"

tests/test_identifiers.py

@@ -30,8 +30,8 @@ def test_check_generic_ror():
         pytest.skip("Cannot Reach ROR API")
     _data = fdp_id._check_generic_ror(ROR_ID)
     assert _data["name"] == "Rakon (France)" == _data["family_name"]
-    assert not "ror" in _data
-    assert not "grid" in _data
+    assert "ror" not in _data
+    assert "grid" not in _data
     assert not fdp_id.check_ror("notanid!")