diff --git a/.flake8 b/.flake8 index de5ca45..a88d78a 100644 --- a/.flake8 +++ b/.flake8 @@ -1,7 +1,16 @@ [flake8] ignore = E501, W503 -max-line-length = 79 +max-line-length = 88 +extend-ignore = E203 max-complexity = 18 select = B,C,E,F,W,T4,B9,N exclude = -**/migrations/** + **/migrations/** + __pycache__ + manage.py + settings.py + env + .env + venv + .venv + tests/data diff --git a/.isort.cfg b/.isort.cfg index e64e0db..b7d98bd 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -6,3 +6,5 @@ force_grid_wrap = 0 use_parentheses = True ensure_newline_before_comments = True line_length = 79 +max-line-length = 88 +extend-ignore = E203 diff --git a/fair/registry/requests.py b/fair/registry/requests.py index 68cf786..d34555e 100644 --- a/fair/registry/requests.py +++ b/fair/registry/requests.py @@ -89,24 +89,13 @@ def _access( method: str = None, token: str = None, obj_path: str = None, - response_codes: typing.List[int] = None, - headers: typing.Dict[str, typing.Any] = None, - params: typing.Dict = None, - data: typing.Dict = None, + response_codes: typing.List[int] = [201, 200], + headers: typing.Dict[str, typing.Any] = {}, + params: typing.Dict = {}, + data: typing.Dict = {}, files: typing.Dict = None, trailing_slash=True, ): - if response_codes is None: - response_codes = [201, 200] - if not headers: - headers: typing.Dict[str, str] = {} - - if not params: - params: typing.Dict[str, str] = {} - - if not data: - data: typing.Dict[str, str] = {} - # Make sure we have the right number of '/' in the components _uri = uri _uri = fdp_util.check_trailing_slash(_uri) @@ -159,7 +148,8 @@ def _access( if _request.status_code == 403: raise fdp_exc.RegistryAPICallError( - f"Failed to run method '{method}' for url {_url}, request forbidden", + f"Failed to run method '{method}' for url {_url}, \ + request forbidden", error_code=403, ) elif _request.status_code == 409: @@ -227,7 +217,10 @@ def post( for param, value in data.copy().items(): if not value: - logger.debug(f"Key in post data '{param}' has no value so will be ignored") + logger.debug( + f"Key in post data '{param}' \ + has no value so will be ignored" + ) del data[param] return _access( @@ -289,7 +282,9 @@ def get( returned data for the given request """ logger.debug( - "Retrieving object of type '%s' from registry at '%s' with parameters: %s", + "Retrieving object of type '%s' \ + from registry at '%s' \ + with parameters: %s", obj_path, uri, params, @@ -303,7 +298,8 @@ def get( for param, value in params.copy().items(): if not value: logger.warning( - f"Key in get parameters '{param}' has no value so will be ignored" + f"Key in get parameters '{param}' \ + has no value so will be ignored" ) del params[param] @@ -367,7 +363,7 @@ def post_else_get( def post_upload_url(remote_uri: str, remote_token: str, file_hash: str) -> str: - """Function to get a tempory url to upload and object to + """Function to get a temporary url to upload and object to Args: remote_uri (str): Remote registry URL @@ -375,7 +371,7 @@ def post_upload_url(remote_uri: str, remote_token: str, file_hash: str) -> str: file_hash (str): Hash of the file to be uploaded Returns: - str: A tempory url to upload the object to + str: A temporary url to upload the object to """ _url = urllib.parse.urljoin(remote_uri, "data/") _url = urllib.parse.urljoin(_url, file_hash) @@ -480,7 +476,8 @@ def put_file(upload_url: str, file_loc: str) -> bool: file_loc (str): Location of the file to be uploaded Raises: - fdp_exc.RegistryError: If the upload fails a RegistryError will be raised + fdp_exc.RegistryError: + If the upload fails a RegistryError will be raised Returns: bool: Will return True if the upload succeeded. @@ -489,7 +486,8 @@ def put_file(upload_url: str, file_loc: str) -> bool: _req = s.put(upload_url, data=open(file_loc, mode="rb").read()) if _req.status_code not in [200, 201]: raise fdp_exc.RegistryError( - f"File: {file_loc} could not be uploaded, Registry Returned: {_req.status_code}" + f"File: {file_loc} could not be uploaded,\ + Registry Returned: {_req.status_code}" ) return True @@ -543,7 +541,8 @@ def download_file(url: str, chunk_size: int = 8192) -> str: def get_dependency_listing( uri: str, token: str, read_only: bool = False ) -> typing.Dict: - """Get complete listing of all objects and their registry based dependencies + """Get complete listing of all objects and + their registry based dependencies Parameters ---------- @@ -648,7 +647,7 @@ def get_author_exists(registry_uri, token=None, name=None, identifier=None) -> s def get_auth_provider(registry_uri: str, token: str = None) -> str: - """Retrived the auth provider from the remote registry + """Retrieve the auth provider from the remote registry Parameters ---------- @@ -667,14 +666,14 @@ def get_auth_provider(registry_uri: str, token: str = None) -> str: if not _response["auth_provider"]: raise fdp_exc.RegistryError( f"The remote registry {registry_uri} \ - did not provide an authentication provider \ - is the remote registry correcly configured?" + did not provide an authentication provider \ + is the remote registry correctly configured?" ) return _response["auth_provider"] def get_auth_url(registry_uri: str, token: str = None) -> str: - """Retrived the auth url from the remote registry + """Retrieve the auth url from the remote registry Parameters ---------- @@ -693,7 +692,7 @@ def get_auth_url(registry_uri: str, token: str = None) -> str: if not _response["auth_url"]: raise fdp_exc.RegistryError( f"The remote registry {registry_uri} \ - did not provide an authentication provider \ - is the remote registry correcly configured?" + did not provide an authentication provider \ + is the remote registry correctly configured?" ) return _response["auth_url"] diff --git a/fair/registry/storage.py b/fair/registry/storage.py index 60e23fb..25ac705 100644 --- a/fair/registry/storage.py +++ b/fair/registry/storage.py @@ -59,7 +59,8 @@ def get_write_storage(uri: str, write_data_store: str, token: str) -> str: Raises ------ fdp_exc.UserConfigError - If 'write_data_store' not present in the working config or global config + If 'write_data_store' not present + in the working config or global config """ logger.debug("Constructing a storage root for '%s'", write_data_store) @@ -256,7 +257,8 @@ def store_working_config(repo_dir: str, uri: str, work_cfg_yml: str, token: str) raise e else: raise fdp_exc.RegistryAPICallError( - f"Cannot post storage_location '{_rel_path}' with hash '{_hash}', object already exists", + f"Cannot post storage_location '{_rel_path}' \ + with hash '{_hash}', object already exists", error_code=409, ) from e @@ -618,7 +620,8 @@ def _get_url_from_data_product( _name = data["data_product"] else: raise fdp_exc.UserConfigError( - f"Failed to determine type while storing item '{label}'" "into registry" + f"Failed to determine type while storing item '{label}' \ + into registry" ) if "data_product" in data["use"]: @@ -640,7 +643,8 @@ def _get_url_from_data_product( raise e else: raise fdp_exc.RegistryAPICallError( - f"Cannot post data_product '{_name}', duplicate already exists", + f"Cannot post data_product '{_name}', \ + duplicate already exists", error_code=409, ) from e @@ -666,7 +670,7 @@ def _get_identifier_from_data(data: typing.Dict, label: str) -> typing.Dict[str, _identifier["alternate_identifier"] = data["unique_name"] except KeyError as e: raise fdp_exc.UserConfigError( - "No identifier/alternate_identifier given for " f"item '{label}'", + f"No identifier/alternate_identifier given for item '{label}'", hint="You must provide either a URL 'identifier', or " "'unique_name' and 'source_name' keys", ) from e @@ -855,10 +859,12 @@ def upload_remote_file( Args: file_loc (str): Location of the file remote_uri (_type_, optional): Remote Registry URL. Defaults to None. - remote_token (_type_, optional): Remote Registry Token. Defaults to None. + remote_token (_type_, optional): + Remote Registry Token. Defaults to None. Raises: - fdp_exc.FileNotFoundError: If the file does not exist a FileNotFound error will be raised. + fdp_exc.FileNotFoundError: + If the file does not exist a FileNotFound error will be raised. """ if not os.path.exists(file_loc): raise fdp_exc.FileNotFoundError(f"File: {file_loc} does not exist") diff --git a/fair/registry/sync.py b/fair/registry/sync.py index 4d057d8..f19bfef 100644 --- a/fair/registry/sync.py +++ b/fair/registry/sync.py @@ -216,7 +216,8 @@ def sync_dependency_chain( if not fdp_util.is_api_url(dest_uri, _new_url): raise fdp_exc.InternalError( - f"Expected new URL '{_new_url}' to be compatible with destination URI '{dest_uri}'" + f"Expected new URL '{_new_url}' to be compatible with \ + destination URI '{dest_uri}'" ) # Fill the new URLs dictionary with the result @@ -237,7 +238,8 @@ def _get_new_url( local_data_store=None, public=False, ) -> typing.Tuple[typing.Dict, typing.List]: - """Internal Function to return a resgistry entry from the remote registry given an origin entry URL + """Internal Function to return a registry entry from the remote registry + given an origin entry URL If the entry does not exist it will be created Args: @@ -246,15 +248,20 @@ def _get_new_url( dest_uri (str): Url of the destination registry dest_token (str): Token of the destination registry object_url (str): Url of the destination registry entry - new_urls (typing.Dict): If the entry contains registry URLS, a list of remote registry URLs for these - writable_data (typing.Dict): Dictionary of writable fields for the given entity type + new_urls (typing.Dict): If the entry contains registry URLS, + a list of remote registry URLs for these + writable_data (typing.Dict): Dictionary of writable fields for + the given entity type Raises: - fdp_exc.RegistryError: If the given new_urls are not valid for the entry a RegistryError will be raised - fdp_exc.InternalError: If the origin url and the destination URLs are the same an InternalError with be raised + fdp_exc.RegistryError: If the given new_urls are not valid for + the entry a RegistryError will be raised + fdp_exc.InternalError: If the origin url and the destination URLs + are the same an InternalError with be raised Returns: - typing.Tuple[typing.Dict, typing.List]: A tuple containing the destination registry entity + typing.Tuple[typing.Dict, typing.List]: A tuple containing the + destination registry entity """ _new_obj_data: typing.Dict[str, typing.Any] = {} _url_fields: typing.List[str] = [] @@ -304,10 +311,11 @@ def _get_new_url( # If Public is true then any files will be uploaded to the remote registry # If local_data_store is set we're pulling to a local registry if public and not local_data_store: - # The remote data_store storage_root URL should always be the 1st storage_root + # The remote data_store storage_root URL should + # always be the 1st storage_root _remote_storage_root_url = urllib.parse.urljoin(dest_uri, "storage_root/1/") - # If the current objects a storage_root and the root is the first (data_store) - # then simply return the remote + # If the current objects a storage_root and the root + # is the first (data_store) then simply return the remote if _obj_type == "storage_root" and _obj_id == "1": return _remote_storage_root_url # If the current object is a storage_location @@ -376,12 +384,12 @@ def sync_author( ) if not new_urls: raise fdp_exc.RegistryError( - f"Auther {identifier}, could not be pushed to {dest_uri}" + f"Author {identifier}, could not be pushed to {dest_uri}" ) new_author_url = new_urls[current_author_url] if not new_author_url: raise fdp_exc.RegistryError( - f"Auther {identifier}, was not be pushed to {dest_uri}" + f"Author {identifier}, was not be pushed to {dest_uri}" ) return new_author_url diff --git a/fair/run.py b/fair/run.py index ad487ce..e8c6064 100644 --- a/fair/run.py +++ b/fair/run.py @@ -64,7 +64,8 @@ def get_job_hash(job_dir: str) -> str: """ if not os.path.exists(job_dir): raise fdp_exc.FileNotFoundError( - "Failed to find hash for job, " f"directory '{job_dir}' does not exist." + f"Failed to find hash for job,\ + directory '{job_dir}' does not exist." ) _directory = os.path.abspath(job_dir) return hashlib.sha1(_directory.encode("utf-8")).hexdigest() diff --git a/fair/staging.py b/fair/staging.py index a3edd5d..f934546 100644 --- a/fair/staging.py +++ b/fair/staging.py @@ -237,13 +237,8 @@ def _get_code_run_entries(self, local_uri: str, job_dir: str) -> typing.List[str _code_run_urls = [] if ( - os.path.exists - ( - _code_run_file - ) and open( - _code_run_file, - encoding="utf-8" - ).read().strip() + os.path.exists(_code_run_file) + and open(_code_run_file, encoding="utf-8").read().strip() ): self._logger.debug("Found coderuns file, extracting runs") _runs = [ diff --git a/tests/conftest.py b/tests/conftest.py index d4d2da1..44115fb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,6 @@ from pathlib import Path import boto3 -import os from moto.server import ThreadedMotoServer import requests @@ -150,7 +149,7 @@ def pyDataPipeline(tmp_path_factory): @pytest.fixture() def pySimpleModel(tmp_path): _repo_path = tmp_path.__str__() - _repo = git.Repo.clone_from(PYTHON_MODEL_GIT, _repo_path) + _ = git.Repo.clone_from(PYTHON_MODEL_GIT, _repo_path) yield _repo_path diff --git a/tests/test_configuration.py b/tests/test_configuration.py index dabc39a..1087301 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -147,7 +147,7 @@ def test_user_info(mocker: pytest_mock.MockerFixture): if not conf.test_can_be_run(f'{fdp_id.QUERY_URLS["orcid"]}{ORCID_ID}'): warnings.warn("Orcid API Unavailable") pytest.skip("Cannot Reach Orcid API") - _data = fdp_id.check_github("FAIRDataPipeline") + _ = fdp_id.check_github("FAIRDataPipeline") _namepaces = {"input": "ispace", "output": "jbloggs"} _email = "jbloggs@nowhere.com" _github_username = "FAIRDataPipeline" diff --git a/tests/test_identifiers.py b/tests/test_identifiers.py index 1ce53ca..baf522c 100644 --- a/tests/test_identifiers.py +++ b/tests/test_identifiers.py @@ -30,8 +30,8 @@ def test_check_generic_ror(): pytest.skip("Cannot Reach ROR API") _data = fdp_id._check_generic_ror(ROR_ID) assert _data["name"] == "Rakon (France)" == _data["family_name"] - assert not "ror" in _data - assert not "grid" in _data + assert "ror" not in _data + assert "grid" not in _data assert not fdp_id.check_ror("notanid!") diff --git a/tests/test_register.py b/tests/test_register.py index 01fca8d..98c1cb8 100644 --- a/tests/test_register.py +++ b/tests/test_register.py @@ -1,20 +1,10 @@ import os -import pathlib -import typing import click.testing import pytest -import pytest_mock import yaml -import fair.registry.sync as fdp_sync -import fair.registry.server as fdp_serv from fair.cli import cli -from fair.common import FAIR_FOLDER -from fair.registry.requests import get, url_get -from tests.conftest import RegistryTest -from tests.conftest import MotoTestServer -import fair.session as fdp_session import fair.common as fdp_com import fair.testing as fdp_test @@ -55,7 +45,7 @@ def test_register( yaml.dump(_config, open(config_path, "w")) with capsys.disabled(): print(_config) - print(f"\tRUNNING: fair init --debug") + print("\tRUNNING: fair init --debug") with local_registry, remote_registry: _res = _cli_runner.invoke( cli, ["init", "--debug", "--using", config_path], catch_exceptions=True diff --git a/tests/test_sync.py b/tests/test_sync.py index 2bfdbea..5a2497d 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -1,6 +1,5 @@ import os import pathlib -import typing import click.testing import pytest @@ -8,13 +7,10 @@ import yaml import fair.registry.sync as fdp_sync -import fair.registry.server as fdp_serv from fair.cli import cli -from fair.common import FAIR_FOLDER -from fair.registry.requests import get, url_get +from fair.registry.requests import get from tests.conftest import RegistryTest from tests.conftest import MotoTestServer -import fair.session as fdp_session import fair.common as fdp_com import fair.testing as fdp_test @@ -123,7 +119,7 @@ def test_init( yaml.dump(_config, open(config_path, "w")) with capsys.disabled(): print(_config) - print(f"\tRUNNING: fair init --debug") + print("\tRUNNING: fair init --debug") with local_registry, remote_registry: _res = _cli_runner.invoke( cli, ["init", "--debug", "--using", config_path], catch_exceptions=True