Merge pull request #809 from GSA/ckan-2-11-0

CKAN 2.11.0 release

.env

@@ -19,6 +19,7 @@ DB_CKAN_USER=ckan
 DB_CKAN_PASSWORD=pass
 DB_CKAN_DB=ckan
 
+CKAN___SECRET_KEY=CHANGE_ME
 CKAN___BEAKER__SESSION__URL=postgresql://ckan:ckan@db/ckan
 CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
 CKAN___WTF_CSRF_SECRET_KEY=CHANGE_ME
@@ -28,7 +29,7 @@ CKAN___API_TOKEN__JWT__DECODE__SECRET=string:CHANGE_ME
 
 # Put envvars last per envvars doc, https://github.com/okfn/ckanext-envvars#requirements
 # Not including saml2auth
-CKAN__PLUGINS=datagov_inventory datastore xloader stats text_view recline_view s3filestore dcat_usmetadata usmetadata datajson envvars
+CKAN__PLUGINS=datagov_inventory datastore xloader s3filestore dcat_usmetadata usmetadata datajson envvars
 
 CKAN__SITE_TITLE=Inventory
 CKAN__SITE_LOGO=/base/images/inventory-logo.png

.profile

@@ -53,6 +53,7 @@ export CKANEXT__SAML2AUTH__CERT_FILE_PATH=${CONFIG_DIR}/saml2_certificate.pem
 DS_RO_PASSWORD=$(vcap_get_service secrets .credentials.DS_RO_PASSWORD)
 export NEW_RELIC_LICENSE_KEY=$(vcap_get_service secrets .credentials.NEW_RELIC_LICENSE_KEY)
 export CKAN___BEAKER__SESSION__SECRET=$(vcap_get_service secrets .credentials.CKAN___BEAKER__SESSION__SECRET)
+export CKAN___SECRET_KEY=$(vcap_get_service secrets .credentials.CKAN___SECRET_KEY)
 export CKAN___WTF_CSRF_SECRET_KEY=$(vcap_get_service secrets .credentials.CKAN___WTF_CSRF_SECRET_KEY)
 export CKAN___CACHE_DIR=${SHARED_DIR}/cache
 
@@ -80,7 +81,8 @@ export CKANEXT__S3FILESTORE__AWS_SECRET_ACCESS_KEY=$(vcap_get_service s3 .creden
 export CKANEXT__S3FILESTORE__AWS_BUCKET_NAME=$(vcap_get_service s3 .credentials.bucket)
 export CKANEXT__S3FILESTORE__AWS_STORAGE_PATH=datagov/inventory-next
 # xloader uses the same db as datastore
-export CKANEXT__XLOADER__JOBS_DB__URI=$(vcap_get_service datastore .credentials.uri)
+export CKANEXT__XLOADER__API_TOKEN=$(vcap_get_service secrets .credentials.API_TOKEN)
+export CKANEXT__XLOADER__JOBS_DB__URI=$(vcap_get_service db .credentials.uri)
 export CKANEXT__XLOADER__JOBS_DB__URI=${CKANEXT__XLOADER__JOBS_DB__URI/postgres/postgresql}
 
 # Write out any files and directories

.snyk

@@ -72,6 +72,12 @@ ignore:
            Issue created and triaged. GitHub issue:
            https://github.com/GSA/data.gov/issues/4951
         expires: 2025-04-30T16:20:58.017Z
+  SNYK-PYTHON-CKAN-8689473:
+    - '*':
+        reason: >-
+           Issue created and triaged. GitHub issue:
+           https://github.com/GSA/data.gov/issues/5071
+        expires: 2025-03-07T16:20:58.017Z
 patch: {}
 # specify the directories or files to be excludeed from import:
 exclude:

Dockerfile

@@ -1,4 +1,4 @@
-FROM ckan/ckan-dev:2.10.5-py3.10
+FROM ckan/ckan-dev:2.11.0
 # Inherit from here: https://github.com/okfn/docker-ckan/blob/master/ckan-dev/2.10/Dockerfile
 # And then from here: https://github.com/okfn/docker-ckan/blob/master/ckan-base/2.10/Dockerfile
 

ckanext/datagov_inventory/plugin.py

@@ -96,7 +96,6 @@ def get_auth_functions(self):
                 'package_search': restrict_anon_access,
                 'package_show': inventory_package_show,
                 'resource_show': inventory_resource_show,
-                'site_read': restrict_anon_access,
                 'tag_list': restrict_anon_access,
                 'tag_show': restrict_anon_access,
                 'task_status_show': restrict_anon_access,
@@ -119,9 +118,16 @@ def get_blueprint(self):
 def redirect_homepage():
     if current_user.is_authenticated or g.user:
         CKAN_SITE_URL = config.get("ckan.site_url")
-        return redirect(CKAN_SITE_URL + '/dataset', code=302)
+        return redirect(CKAN_SITE_URL + '/dataset/', code=302)
     else:
         return base.render(u'error/anonymous.html')
 
 
 pusher.add_url_rule('/', view_func=redirect_homepage)
+
+
+@pusher.before_app_request
+def check_dataset_access():
+    if toolkit.request.path in ('/dataset/', '/dataset'):
+        if not current_user.is_authenticated and not g.user:
+            return base.render(u'error/anonymous.html'), 403

ckanext/datagov_inventory/tests/logic/auth/test_auth.py

@@ -313,19 +313,6 @@ def test_auth_resource_show_for_public_gsa_dataset(self):
             'anonymous': is_allowed
         }, object_id=dataset['resource_id'])
 
-    def test_auth_site_read(self):
-        # Create test users and test data
-        self.setup_test_orgs_users()
-
-        self.assert_user_authorization('site_read', {
-            'gsa_admin': is_allowed,
-            'gsa_editor': is_allowed,
-            'gsa_member': is_allowed,
-            'doi_admin': is_allowed,
-            'doi_member': is_allowed,
-            'anonymous': is_denied
-        })
-
     def test_auth_tag_list(self):
         # Create test users and test data
         self.setup_test_orgs_users()

config/ckan.ini

@@ -147,7 +147,7 @@ ckan.cors.origin_whitelist = https://gsa.gov https://www.gsa.gov
 #		same origin policy
 
 # Put envvars last per envvars doc, https://github.com/okfn/ckanext-envvars#requirements
-ckan.plugins = datagov_inventory datastore xloader stats text_view recline_view s3filestore dcat_usmetadata usmetadata datajson saml2auth envvars
+ckan.plugins = datagov_inventory datastore xloader s3filestore dcat_usmetadata usmetadata datajson saml2auth envvars
 
 ckan.views.default_views = recline_view text_view image_view webpage_view recline_grid_view
 

docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   ckan:
-    image: datagov/inventory-app:2.10.5
+    image: datagov/inventory-app:2.11.0
     build: .
     command: /app/start.sh
     depends_on:

e2e/cypress/integration/ckan_extensions.cy.js

@@ -1,8 +1,8 @@
 describe('CKAN Extensions', () => {
-    it('Uses CKAN 2.10', () => {
+    it('Uses CKAN 2.11', () => {
         cy.request('/api/action/status_show').should((response) => {
             expect(response.body).to.have.property('success', true);
-            expect(response.body.result).to.have.property('ckan_version', '2.10.5');
+            expect(response.body.result).to.have.property('ckan_version', '2.11.0');
         });
     });
 
@@ -12,8 +12,6 @@ describe('CKAN Extensions', () => {
             const installed_extensions = response.body.result.extensions;
             expect(installed_extensions).to.include('datastore');
             expect(installed_extensions).to.include('xloader');
-            expect(installed_extensions).to.include('stats');
-            expect(installed_extensions).to.include('recline_view');
             expect(installed_extensions).to.include('s3filestore');
             expect(installed_extensions).to.include('envvars');
             expect(installed_extensions).to.include('datastore');

e2e/cypress/integration/public_access.cy.js

@@ -26,6 +26,13 @@ describe('Public Access', () => {
         }).then((response) => {
             expect(response.status).to.eq(403);
         });
+
+        cy.request({
+            url: '/dataset/',
+            failOnStatusCode: false,
+        }).then((response) => {
+            expect(response.status).to.eq(403);
+        });
     });
 
     it('Cannot access the dataset pages', () => {

requirements.in.txt

@@ -1,10 +1,10 @@
-git+https://github.com/GSA/ckan.git@ckan-2-10-5-fork#egg=ckan
+git+https://github.com/ckan/ckan.git@ckan-2.11.0#egg=ckan
 # TODO https://github.com/GSA/datagov-deploy/issues/2794
-git+https://github.com/GSA/ckanext-saml2auth.git@datagov#egg=ckanext-saml2auth
+git+https://github.com/GSA/ckanext-saml2auth.git@ckan-2-11#egg=ckanext-saml2auth
 git+https://github.com/keitaroinc/ckanext-s3filestore.git#egg=ckanext-s3filestore
 -e git+https://github.com/ckan/[email protected]#egg=ckanext-xloader
 
-ckanext-usmetadata>=0.3.2
+ckanext-usmetadata>=0.3.3
 ckanext-datajson>=0.1.19
 ckanext-dcat-usmetadata~=0.6.0
 ckanext-envvars>=0.0.3
@@ -17,45 +17,41 @@ lxml==4.9.1
 # Jinja2==2.11.3
 
 # Add necessary CKAN core libraries
-alembic==1.8.1
-Babel==2.10.3
-backports-zoneinfo==0.2.1; python_version < '3.9'
-Beaker==1.11.0
-bleach==5.0.1
-blinker==1.5
-click==8.1.3
-dominate==2.7.0
-feedgen==0.9.0
-Flask==2.0.3
-Flask-Babel==1.0.0
-Flask-Login==0.6.1
-Flask-WTF==1.0.1
-flask-multistatic==1.0
-# greenlet==2.0.2
-# Jinja2==3.1.2
-Markdown==3.4.1
+alembic==1.13.2
+Babel==2.15.0
+bleach==6.1.0
+blinker==1.8.2
+dominate==2.9.1
+feedgen==1.0.0
+Flask==3.0.3
+Flask-Babel==4.0.0
+Flask-Login==0.6.3
+Flask-Session==0.8.0
+Flask-WTF==1.2.1
+Jinja2>=3.1.5
+Markdown==3.6
+msgspec==0.18.6
 packaging==24.1
 passlib==1.7.4
-polib==1.1.1
-psycopg2==2.9.3
-PyJWT==2.4.0
+polib==1.2.0
+psycopg2==2.9.9
+PyJWT==2.8.0
 pyparsing==3.1.2
 python-magic==0.4.27
 pysolr==3.9.0
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
 pytz
-PyUtilib==6.0.0
 pyyaml==6.0.1
-requests~=2.32.3
-rq==1.11.0
-simplejson==3.18.0
-SQLAlchemy[mypy]==1.4.41
+requests==2.32.3
+rq==1.16.2
+simplejson==3.19.2
+SQLAlchemy[mypy]==1.4.52
 sqlparse==0.5.0
-typing_extensions==4.3.0
-tzlocal==4.2
+typing_extensions==4.12.2
+tzlocal==5.2
 webassets==2.0
-Werkzeug[watchdog]==2.0.3
-zope.interface==5.4.0
+Werkzeug[watchdog]==3.0.3
+zope.interface==6.4post2
 
 # # ckanext-saml2 dependencies
 # defusedxml==0.6.0
@@ -77,23 +73,16 @@ python-dateutil>=2.8.2
 
 # Other pinned dependencies
 # (most likely for snyk)
-itsdangerous==2.0.1
-MarkupSafe==2.0.1
 wheel==0.42.0
 
 # avoid ImportError error https://github.com/GSA/data.gov/issues/4396
 importlib-resources<6.0
-cryptography>42.0.4
-jinja2>=3.1.5
-
-# fix for https://security.snyk.io/vuln/SNYK-PYTHON-GEVENT-8320934
 gevent>=24.10.1
+cryptography>42.0.4
+pip>=23.3
 
 pyparsing # need to avoid solr missing module error on cloud.gov
 
-# duplicate Werkzeug[watchdog]==2.0.3 requrements from ckan list
-# so that snyk doesn't complain about the version mismatch
-Werkzeug==2.0.3
 urllib3>=1.26.19
 
 certifi>=2024.7.4
@@ -104,6 +93,4 @@ setuptools~=71.0.3
 # https://github.com/GSA/data.gov/issues/4954
 MarkupSafe==2.*
 
-# avoid conflic dependencies issue
-greenlet>=3.1.1
 pip>=25.0