From 340158e34343b9bec943e08d7a6b5318f35c6e5b Mon Sep 17 00:00:00 2001 From: David Linke <dr.david.linke@gmail.com> Date: Wed, 8 Jan 2025 00:01:06 +0100 Subject: [PATCH 1/8] Simplify makefile and config --- {{cookiecutter.project_name}}/Makefile | 9 +++++---- {{cookiecutter.project_name}}/config.public.mk | 4 ++-- {{cookiecutter.project_name}}/config.yaml | 2 ++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/{{cookiecutter.project_name}}/Makefile b/{{cookiecutter.project_name}}/Makefile index 3a4bc5f..2790883 100644 --- a/{{cookiecutter.project_name}}/Makefile +++ b/{{cookiecutter.project_name}}/Makefile @@ -24,7 +24,7 @@ PYMODEL = $(SRC)/$(SCHEMA_NAME)/datamodel DOCDIR = docs DOCTEMPLATES = $(SRC)/docs/templates EXAMPLEDIR = examples -SHEET_MODULE = {{cookiecutter.__google_sheet_module}} +SHEET_MODULE = $(LINKML_SCHEMA_GOOGLE_SHEET_MODULE) SHEET_ID = $(LINKML_SCHEMA_GOOGLE_SHEET_ID) SHEET_TABS = $(LINKML_SCHEMA_GOOGLE_SHEET_TABS) SHEET_MODULE_PATH = $(SOURCE_SCHEMA_DIR)/$(SHEET_MODULE).yaml @@ -32,6 +32,7 @@ SHEET_MODULE_PATH = $(SOURCE_SCHEMA_DIR)/$(SHEET_MODULE).yaml # Use += to append variables from the variables file CONFIG_YAML = ifdef LINKML_GENERATORS_CONFIG_YAML +CONFIG_YAML += "--config-file" CONFIG_YAML += ${LINKML_GENERATORS_CONFIG_YAML} endif @@ -151,7 +152,7 @@ test-schema: $(RUN) gen-project ${CONFIG_YAML} -d tmp $(SOURCE_SCHEMA_PATH) test-python: - $(RUN) python -m unittest discover + $(RUN) python -m pytest lint: $(RUN) linkml-lint $(SOURCE_SCHEMA_PATH) @@ -175,7 +176,7 @@ examples/%.ttl: src/data/examples/%.yaml test-examples: examples/output -examples/output: src/{{cookiecutter.__project_slug}}/schema/{{cookiecutter.__project_slug}}.yaml +examples/output: src/$(SCHEMA_NAME)/schema/$(SCHEMA_NAME).yaml mkdir -p $@ $(RUN) linkml-run-examples \ --output-formats json \ @@ -224,7 +225,7 @@ git-status: clean: rm -rf $(DEST) rm -rf tmp - rm -fr docs/* + rm -fr $(DOCDIR)/* rm -fr $(PYMODEL)/* include project.Makefile diff --git a/{{cookiecutter.project_name}}/config.public.mk b/{{cookiecutter.project_name}}/config.public.mk index ab92906..af7325c 100644 --- a/{{cookiecutter.project_name}}/config.public.mk +++ b/{{cookiecutter.project_name}}/config.public.mk @@ -11,13 +11,14 @@ LINKML_SCHEMA_NAME={{cookiecutter.__project_slug}} LINKML_SCHEMA_AUTHOR={{cookiecutter.__author}} LINKML_SCHEMA_DESCRIPTION={{cookiecutter.project_description}} LINKML_SCHEMA_SOURCE_PATH={{cookiecutter.__source_path}} +LINKML_SCHEMA_GOOGLE_SHEET_MODULE={{cookiecutter.__google_sheet_module}} LINKML_SCHEMA_GOOGLE_SHEET_ID={{cookiecutter.google_sheet_id}} LINKML_SCHEMA_GOOGLE_SHEET_TABS={{cookiecutter.google_sheet_tabs}} ###### linkml generator variables, used by makefile ## gen-project configuration file -LINKML_GENERATORS_CONFIG_YAML= --config-file config.yaml +LINKML_GENERATORS_CONFIG_YAML=config.yaml ## pass args if gendoc ignores config.yaml (i.e. --no-mergeimports) LINKML_GENERATORS_DOC_ARGS= @@ -29,4 +30,3 @@ LINKML_GENERATORS_OWL_ARGS= ## pass args to trigger experimental java/typescript generation LINKML_GENERATORS_JAVA_ARGS= LINKML_GENERATORS_TYPESCRIPT_ARGS= - diff --git a/{{cookiecutter.project_name}}/config.yaml b/{{cookiecutter.project_name}}/config.yaml index 56da66d..ddc6cb4 100644 --- a/{{cookiecutter.project_name}}/config.yaml +++ b/{{cookiecutter.project_name}}/config.yaml @@ -22,6 +22,8 @@ generator_args: mergeimports: true jsonldcontext: mergeimports: true + pydantic: + mergeimports: true python: mergeimports: true prefixmap: From e1235847c3172202dd511eca316e88d1fde688ba Mon Sep 17 00:00:00 2001 From: David Linke <dr.david.linke@gmail.com> Date: Wed, 8 Jan 2025 00:03:04 +0100 Subject: [PATCH 2/8] Add a justfile matching the make commands --- {{cookiecutter.project_name}}/justfile | 196 ++++++++++++++++++ .../project.justfile | 1 + 2 files changed, 197 insertions(+) create mode 100644 {{cookiecutter.project_name}}/justfile create mode 100644 {{cookiecutter.project_name}}/project.justfile diff --git a/{{cookiecutter.project_name}}/justfile b/{{cookiecutter.project_name}}/justfile new file mode 100644 index 0000000..0ff37bd --- /dev/null +++ b/{{cookiecutter.project_name}}/justfile @@ -0,0 +1,196 @@ +# On Windows the bash shell that comes with Git for Windows should be used. +# If it is not on path, give the path to the executable in the following line. +#set windows-shell := ["C:/Program Files/Git/usr/bin/sh", "-cu"] + +# Load environment variables from config.public.mk or specified file +set dotenv-load := true +# set dotenv-filename := env_var_or_default("LINKML_ENVIRONMENT_FILENAME", "config.public.mk") +set dotenv-filename := x'${LINKML_ENVIRONMENT_FILENAME:-config.public.mk}' + + +# List all commands as default command. The prefix "_" hides the command. +_default: _status + @just --list + +# Set cross-platform Python shebang line (assumes presence of launcher on Windows) +shebang := if os() == 'windows' { + 'py' +} else { + '/usr/bin/env python3' +} + +# Environment variables with defaults +schema_name := env_var_or_default("LINKML_SCHEMA_NAME", "") +source_schema_path := env_var_or_default("LINKML_SCHEMA_SOURCE_PATH", "") + +sheet_module := env_var_or_default("LINKML_SCHEMA_GOOGLE_SHEET_MODULE", "") +sheet_ID := env_var_or_default("LINKML_SCHEMA_GOOGLE_SHEET_ID", "") +sheet_tabs := env_var_or_default("LINKML_SCHEMA_GOOGLE_SHEET_TABS", "") +sheet_module_path := source_schema_path / sheet_module + ".yaml" + +config_yaml := if env_var_or_default("LINKML_GENERATORS_CONFIG_YAML", "") == "" { + "--config-file " + env_var_or_default("LINKML_GENERATORS_CONFIG_YAML", "") +} else { + "" +} +gen_doc_args := env_var_or_default("LINKML_GENERATORS_DOC_ARGS", "") +gen_owl_args := env_var_or_default("LINKML_GENERATORS_OWL_ARGS", "") +gen_java_args := env_var_or_default("LINKML_GENERATORS_JAVA_ARGS", "") +gen_ts_args := env_var_or_default("LINKML_GENERATORS_TYPESCRIPT_ARGS", "") + +# Directory variables +src := "src" +dest := "project" +pymodel := src / schema_name / "datamodel" +docdir := "docs" +exampledir := "examples" + +# Show current project status +_status: _check-config + @echo "Project: {{schema_name}}" + @echo "Source: {{source_schema_path}}" + +# Run initial setup (run this first) +setup: _check-config _git-init install _gen-project _gen-examples _gendoc _git-add _git-commit + +# Install project dependencies +install: + poetry install + +# Check project configuration +_check-config: + #!{{shebang}} + import os + schema_name = os.getenv('LINKML_SCHEMA_NAME') + if not schema_name: + print('**Project not configured**:\n - See \'.env.public\'') + exit(1) + print('Project-status: Ok') + +# Updates project template and LinkML package +update: _update-template _update-linkml + +# Update project template +_update-template: + cruft update + +# Update LinkML to latest version +_update-linkml: + poetry add -D linkml@latest + +# Create data harmonizer +_create-data-harmonizer: + npm init data-harmonizer {{source_schema_path}} + +# Generate all project files +alias all := site + +# Generate site locally +site: _gen-project _gendoc + +# Deploy site +deploy: site + mkd-gh-deploy + +_compile_sheets: + @if [ ! -z "${{sheet_module}}" ]; then \ + poetry run sheets2linkml --gsheet-id {{sheet_ID}} {{sheet_tabs}} > {{sheet_module_path}}.tmp && \ + mv {{sheet_module_path}}.tmp {{sheet_module_path}}; \ + fi + +# Generate examples +_gen-examples: + mkdir -p {{exampledir}} + cp src/data/examples/* {{exampledir}} + +# Generate project files +_gen-project: _ensure_pymodel_dir _compile_sheets + poetry run gen-project {{config_yaml}} -d {{dest}} {{source_schema_path}} + mv {{dest}}/*.py {{pymodel}} + poetry run gen-pydantic {{source_schema_path}} > "{{pymodel}}/{{schema_name}}_pydantic.py" + @if [ ! -z "${{gen_owl_args}}" ]; then \ + poetry run gen-owl {{gen_owl_args}} {{source_schema_path}} > {{dest}}/owl/{{schema_name}}.owl.ttl || true && \ + poetry run gen-owl {{gen_owl_args}} {{source_schema_path}} > {{dest}}/owl/{{schema_name}}.owl.ttl || true ; \ + fi + @if [ ! ${{gen_java_args}} ]; then \ + poetry run gen-java {{gen_java_args}} --output-directory {{dest}}/java/ {{source_schema_path}} || true ; \ + fi + @if [ ! ${{gen_ts_args}} ]; then \ + poetry run gen-typescript {{gen_ts_args}} {{source_schema_path}} > {{dest}}/typescript/{{schema_name}}.ts || true ; \ + fi + +# Run all tests +test: _test-schema _test-python _test-examples + +# Test schema generation +_test-schema: + poetry run gen-project {{config_yaml}} -d tmp {{source_schema_path}} + +# Run Python unit tests +_test-python: + poetry run python -m unittest discover + +# Run example tests +_test-examples: _ensure_examples_output + poetry run linkml-run-examples \ + --output-formats json \ + --output-formats yaml \ + --counter-example-input-directory src/data/examples/invalid \ + --input-directory src/data/examples/valid \ + --output-directory examples/output \ + --schema src/pid4cat_model/schema/pid4cat_model.yaml > examples/output/README.md + +# Run linting +lint: + poetry run linkml-lint {{source_schema_path}} + +# Generate documentation +_gendoc: _ensure_docdir + cp -r {{src}}/docs/files/* {{docdir}} + poetry run gen-doc {{gen_doc_args}} -d {{docdir}} {{source_schema_path}} + +# Build docs and run test server +testdoc: _gendoc _serve + +# Run documentation server +_serve: + poetry run mkdocs serve + +# Initialize and add everything to git +_git-init-add: _git-init _git-add _git-commit _git-status + +# Initialize git repository +_git-init: + git init + +# Add files to git +_git-add: + touch .cruft.json + git add . + +# Commit files to git +_git-commit: + git commit -m 'chore: make setup was run' -a + +# Show git status +_git-status: + git status + +# Clean all generated files +clean: + rm -rf {{dest}} + rm -rf tmp + rm -rf {{docdir}}/* + rm -rf {{pymodel}} + +# Private recipes +_ensure_pymodel_dir: + -mkdir -p {{pymodel}} + +_ensure_docdir: + -mkdir -p {{docdir}} + +_ensure_examples_output: + -mkdir -p examples/output + +import "project.justfile" diff --git a/{{cookiecutter.project_name}}/project.justfile b/{{cookiecutter.project_name}}/project.justfile new file mode 100644 index 0000000..bf84fbc --- /dev/null +++ b/{{cookiecutter.project_name}}/project.justfile @@ -0,0 +1 @@ +## Add your own just recipes here. This is imported by the main justfile. From 82fae5e1ad91e7fe1f418900f924b1567b10fadf Mon Sep 17 00:00:00 2001 From: David Linke <dr.david.linke@gmail.com> Date: Wed, 8 Jan 2025 00:12:00 +0100 Subject: [PATCH 3/8] Update READMEs for just --- README.md | 8 ++++++++ {{cookiecutter.project_name}}/README.md | 9 +++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 892b59c..817faac 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,14 @@ The following are required and recommended tools for using this cookiecutter and ``` You may also choose to not have a persistent installation of cruft, in which case you would replace any calls to the `cruft` command below with `pipx run cruft`. + + * **make or just as command runner** + + The project contains a makefile but also a `justfile` with pre-defined complex commands. To execute these commands you either need `make` or [just](https://github.com/casey/just) as an alternative command runner. Especially for Windows users we suggest `just`. Install it by running: + ```shell + pipx install just + ``` + ## Creating a new project ### Step 1: Generate the project files diff --git a/{{cookiecutter.project_name}}/README.md b/{{cookiecutter.project_name}}/README.md index 6d7742d..82e3411 100644 --- a/{{cookiecutter.project_name}}/README.md +++ b/{{cookiecutter.project_name}}/README.md @@ -23,10 +23,11 @@ ## Developer Documentation <details> -Use the `make` command to generate project artefacts: - -* `make all`: make everything -* `make deploy`: deploys site +To run commands you may use good old make or the command runner [just](https://github.com/casey/just/) which is a better choice on Windows. +Use the `make` command or `duty` commands to generate project artefacts: +* `make help` or `just --list`: list all pre-defined tasks +* `make all` or `just all`: make everything +* `make deploy` or `just deploy`: deploys site </details> ## Credits From d20d919ab3a196de72a70ca0bf6f3a58327f4e68 Mon Sep 17 00:00:00 2001 From: David Linke <dr.david.linke@gmail.com> Date: Sat, 11 Jan 2025 00:28:11 +0100 Subject: [PATCH 4/8] Don't parse justfile with cookiecutter --- cookiecutter.json | 1 + 1 file changed, 1 insertion(+) diff --git a/cookiecutter.json b/cookiecutter.json index 0aff389..cc8fbd8 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -19,6 +19,7 @@ "__google_sheet_module": "{{ cookiecutter.google_sheet_tabs|lower()|replace(' ', '_')|replace('-', '_') }}", "github_token_for_pypi_deployment": "PYPI_PASSWORD", "_copy_without_render": [ + "justfile", "src/docs/js/*", "src/docs/javascript/*" ] From 239051f22e4bc06a8e4b907adeb323c135c338c7 Mon Sep 17 00:00:00 2001 From: David Linke <dr.david.linke@gmail.com> Date: Sat, 11 Jan 2025 00:44:04 +0100 Subject: [PATCH 5/8] Quote values in dotenv file --- {{cookiecutter.project_name}}/config.public.mk | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/{{cookiecutter.project_name}}/config.public.mk b/{{cookiecutter.project_name}}/config.public.mk index af7325c..064cd79 100644 --- a/{{cookiecutter.project_name}}/config.public.mk +++ b/{{cookiecutter.project_name}}/config.public.mk @@ -7,13 +7,13 @@ ###### schema definition variables, used by makefile # Note: makefile variables should not be quoted, as makefile handles quoting differently than bash -LINKML_SCHEMA_NAME={{cookiecutter.__project_slug}} -LINKML_SCHEMA_AUTHOR={{cookiecutter.__author}} -LINKML_SCHEMA_DESCRIPTION={{cookiecutter.project_description}} -LINKML_SCHEMA_SOURCE_PATH={{cookiecutter.__source_path}} -LINKML_SCHEMA_GOOGLE_SHEET_MODULE={{cookiecutter.__google_sheet_module}} -LINKML_SCHEMA_GOOGLE_SHEET_ID={{cookiecutter.google_sheet_id}} -LINKML_SCHEMA_GOOGLE_SHEET_TABS={{cookiecutter.google_sheet_tabs}} +LINKML_SCHEMA_NAME="{{cookiecutter.__project_slug}}" +LINKML_SCHEMA_AUTHOR="{{cookiecutter.__author}}" +LINKML_SCHEMA_DESCRIPTION="{{cookiecutter.project_description}}" +LINKML_SCHEMA_SOURCE_PATH="{{cookiecutter.__source_path}}" +LINKML_SCHEMA_GOOGLE_SHEET_MODULE="{{cookiecutter.__google_sheet_module}}" +LINKML_SCHEMA_GOOGLE_SHEET_ID="{{cookiecutter.google_sheet_id}}" +LINKML_SCHEMA_GOOGLE_SHEET_TABS="{{cookiecutter.google_sheet_tabs}}" ###### linkml generator variables, used by makefile From 846de5763167922ccbda057dce8dd161bad2b49c Mon Sep 17 00:00:00 2001 From: David Linke <dr.david.linke@gmail.com> Date: Sat, 11 Jan 2025 01:28:05 +0100 Subject: [PATCH 6/8] Fixes after testing all commands --- {{cookiecutter.project_name}}/config.public.mk | 1 + {{cookiecutter.project_name}}/justfile | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/{{cookiecutter.project_name}}/config.public.mk b/{{cookiecutter.project_name}}/config.public.mk index 064cd79..add1b6e 100644 --- a/{{cookiecutter.project_name}}/config.public.mk +++ b/{{cookiecutter.project_name}}/config.public.mk @@ -14,6 +14,7 @@ LINKML_SCHEMA_SOURCE_PATH="{{cookiecutter.__source_path}}" LINKML_SCHEMA_GOOGLE_SHEET_MODULE="{{cookiecutter.__google_sheet_module}}" LINKML_SCHEMA_GOOGLE_SHEET_ID="{{cookiecutter.google_sheet_id}}" LINKML_SCHEMA_GOOGLE_SHEET_TABS="{{cookiecutter.google_sheet_tabs}}" +LINKML_USE_SCHEMASHEETS={{cookiecutter.use_schemasheets}} ###### linkml generator variables, used by makefile diff --git a/{{cookiecutter.project_name}}/justfile b/{{cookiecutter.project_name}}/justfile index 0ff37bd..dd0a5df 100644 --- a/{{cookiecutter.project_name}}/justfile +++ b/{{cookiecutter.project_name}}/justfile @@ -23,12 +23,13 @@ shebang := if os() == 'windows' { schema_name := env_var_or_default("LINKML_SCHEMA_NAME", "") source_schema_path := env_var_or_default("LINKML_SCHEMA_SOURCE_PATH", "") +use_schemasheets := env_var_or_default("LINKML_USE_SCHEMASHEETS", "No") sheet_module := env_var_or_default("LINKML_SCHEMA_GOOGLE_SHEET_MODULE", "") sheet_ID := env_var_or_default("LINKML_SCHEMA_GOOGLE_SHEET_ID", "") sheet_tabs := env_var_or_default("LINKML_SCHEMA_GOOGLE_SHEET_TABS", "") sheet_module_path := source_schema_path / sheet_module + ".yaml" -config_yaml := if env_var_or_default("LINKML_GENERATORS_CONFIG_YAML", "") == "" { +config_yaml := if env_var_or_default("LINKML_GENERATORS_CONFIG_YAML", "") != "" { "--config-file " + env_var_or_default("LINKML_GENERATORS_CONFIG_YAML", "") } else { "" @@ -93,8 +94,8 @@ deploy: site mkd-gh-deploy _compile_sheets: - @if [ ! -z "${{sheet_module}}" ]; then \ - poetry run sheets2linkml --gsheet-id {{sheet_ID}} {{sheet_tabs}} > {{sheet_module_path}}.tmp && \ + @if [ "{{use_schemasheets}}" != "No" ]; then \ + poetry run sheets2linkml --gsheet-id {{sheet_ID}} {{sheet_tabs}} > {{sheet_module_path}}.tmp && \ mv {{sheet_module_path}}.tmp {{sheet_module_path}}; \ fi @@ -105,11 +106,10 @@ _gen-examples: # Generate project files _gen-project: _ensure_pymodel_dir _compile_sheets - poetry run gen-project {{config_yaml}} -d {{dest}} {{source_schema_path}} + poetry run gen-project {{config_yaml}} -d {{dest}} {{source_schema_path}} && \ mv {{dest}}/*.py {{pymodel}} - poetry run gen-pydantic {{source_schema_path}} > "{{pymodel}}/{{schema_name}}_pydantic.py" @if [ ! -z "${{gen_owl_args}}" ]; then \ - poetry run gen-owl {{gen_owl_args}} {{source_schema_path}} > {{dest}}/owl/{{schema_name}}.owl.ttl || true && \ + mkdir -p {{dest}}/owl || true && \ poetry run gen-owl {{gen_owl_args}} {{source_schema_path}} > {{dest}}/owl/{{schema_name}}.owl.ttl || true ; \ fi @if [ ! ${{gen_java_args}} ]; then \ @@ -126,9 +126,9 @@ test: _test-schema _test-python _test-examples _test-schema: poetry run gen-project {{config_yaml}} -d tmp {{source_schema_path}} -# Run Python unit tests +# Run Python unit tests with pytest _test-python: - poetry run python -m unittest discover + poetry run python -m pytest # Run example tests _test-examples: _ensure_examples_output @@ -138,7 +138,7 @@ _test-examples: _ensure_examples_output --counter-example-input-directory src/data/examples/invalid \ --input-directory src/data/examples/valid \ --output-directory examples/output \ - --schema src/pid4cat_model/schema/pid4cat_model.yaml > examples/output/README.md + --schema {{source_schema_path}} > examples/output/README.md # Run linting lint: From 3eb2522f5baa9e8f27ffb4ae28c0134a42d72c9d Mon Sep 17 00:00:00 2001 From: David Linke <dr.david.linke@gmail.com> Date: Sat, 11 Jan 2025 13:28:08 +0100 Subject: [PATCH 7/8] fixup! Simplify makefile and config --- {{cookiecutter.project_name}}/config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/{{cookiecutter.project_name}}/config.yaml b/{{cookiecutter.project_name}}/config.yaml index ddc6cb4..56da66d 100644 --- a/{{cookiecutter.project_name}}/config.yaml +++ b/{{cookiecutter.project_name}}/config.yaml @@ -22,8 +22,6 @@ generator_args: mergeimports: true jsonldcontext: mergeimports: true - pydantic: - mergeimports: true python: mergeimports: true prefixmap: From ed1ceb52e5d6b5a187e2c8d570d3b53e85bd8393 Mon Sep 17 00:00:00 2001 From: David Linke <dr.david.linke@gmail.com> Date: Fri, 31 Jan 2025 00:48:58 +0100 Subject: [PATCH 8/8] Fix pipx install command for just --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 817faac..0b90b0e 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ The following are required and recommended tools for using this cookiecutter and The project contains a makefile but also a `justfile` with pre-defined complex commands. To execute these commands you either need `make` or [just](https://github.com/casey/just) as an alternative command runner. Especially for Windows users we suggest `just`. Install it by running: ```shell - pipx install just + pipx install rust-just ``` ## Creating a new project