cmd-diff: support lightweight ostree import for RPM diff

pytest.ini

@@ -1,2 +1,2 @@
 [pytest]
-addopts = --cov=cosalib.cli --cov=cosalib.meta --cov=cosalib.cmdlib --cov-report term --cov-fail-under=70
+addopts = --cov=cosalib.cli --cov=cosalib.meta --cov=cosalib.cmdlib --cov-report term

src/cmd-diff

@@ -8,6 +8,7 @@ import sys
 import tempfile
 
 from dataclasses import dataclass
+from enum import IntEnum
 from typing import Callable
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
@@ -27,11 +28,17 @@ class DiffBuildTarget:
                                builds.get_build_meta(build, arch))
 
 
+class OSTreeImport(IntEnum):
+    NO = 1
+    PARTIAL = 2
+    FULL = 3
+
+
 @dataclass
 class Differ:
     name: str
     description: str
-    needs_ostree: bool
+    needs_ostree: OSTreeImport
     function: Callable[[DiffBuildTarget, DiffBuildTarget], None]
 
 
@@ -76,9 +83,11 @@ def main():
             active_differs += [differ]
 
     # ensure commits are imported if we know we'll need them
-    if any(differ.needs_ostree for differ in active_differs):
+    ostree_import = max([d.needs_ostree for d in active_differs])
+    if ostree_import > OSTreeImport.NO:
         for target in [diff_from, diff_to]:
-            import_ostree_commit('.', target.dir, target.meta, extract_json=0)
+            import_ostree_commit('.', target.dir, target.meta, extract_json=False,
+                                 partial_import=(ostree_import == OSTreeImport.PARTIAL))
 
     # start diff'ing
     for differ in active_differs:
@@ -104,6 +113,33 @@ def parse_args():
     return parser.parse_args()
 
 
+def diff_source_control(diff_from, diff_to):
+    for source in ['coreos-assembler.container-config-git', 'coreos-assembler.container-image-git']:
+        config_from = diff_from.meta[source]
+        config_to = diff_to.meta[source]
+        config_shared_history = True
+        for item in ['origin', 'branch', 'dirty']:
+            if config_from[item] != config_to[item]:
+                config_shared_history = False
+                break
+        if not config_shared_history:
+            # If they weren't from the same repo/branch, etc then
+            # there's not really any way to compare them easily
+            # so just output the details gory details and move on.
+            print(f"from: {config_from}")
+            print(f"to: {config_to}")
+        else:
+            print(f"{config_from['origin']}: {config_from['commit'][:7]}..{config_to['commit'][:7]}")
+            # If the git repo is on github (which our repos are) let's print a link
+            # where a user can click (or share) and view the changes from one commit
+            # to another.
+            if 'github.com' in config_from['origin']:
+                # Also pull off `.git` if it is on the end of the URL since the
+                # compare API won't work if `.git` is in there.
+                origin_url = f"{config_from['origin']}".removesuffix('.git')
+                print(f"    --> {origin_url}/compare/{config_from['commit'][:7]}...{config_to['commit'][:7]}")
+
+
 def diff_rpms(diff_from, diff_to):
     commit_from = diff_from.meta['ostree-commit']
     commit_to = diff_to.meta['ostree-commit']
@@ -295,29 +331,31 @@ def cache_dir(dir):
 
 # unfortunately, this has to come at the end to resolve functions
 DIFFERS = [
-    Differ("rpms", "Diff RPMs", needs_ostree=True, function=diff_rpms),
+    Differ("rpms", "Diff RPMs", needs_ostree=OSTreeImport.PARTIAL, function=diff_rpms),
+    Differ("source-control", "Diff config and COSA input commits",
+           needs_ostree=OSTreeImport.NO, function=diff_source_control),
     Differ("ostree-ls", "Diff OSTree contents using 'ostree diff'",
-           needs_ostree=True, function=diff_ostree_ls),
+           needs_ostree=OSTreeImport.FULL, function=diff_ostree_ls),
     Differ("ostree", "Diff OSTree contents using 'git diff'",
-           needs_ostree=True, function=diff_ostree),
+           needs_ostree=OSTreeImport.FULL, function=diff_ostree),
     Differ("initrd", "Diff initramfs contents",
-           needs_ostree=True, function=diff_initrd),
+           needs_ostree=OSTreeImport.FULL, function=diff_initrd),
     Differ("live-iso-ls", "Diff live ISO listings",
-           needs_ostree=False, function=diff_live_iso_tree),
+           needs_ostree=OSTreeImport.NO, function=diff_live_iso_tree),
     Differ("live-iso", "Diff live ISO content",
-           needs_ostree=False, function=diff_live_iso),
+           needs_ostree=OSTreeImport.NO, function=diff_live_iso),
     Differ("live-initrd-ls", "Diff live initramfs listings",
-           needs_ostree=False, function=diff_live_initrd_tree),
+           needs_ostree=OSTreeImport.NO, function=diff_live_initrd_tree),
     Differ("live-initrd", "Diff live initramfs content",
-           needs_ostree=False, function=diff_live_initrd),
+           needs_ostree=OSTreeImport.NO, function=diff_live_initrd),
     Differ("live-rootfs-img-ls", "Diff live-rootfs.img listings",
-           needs_ostree=False, function=diff_live_rootfs_img_tree),
+           needs_ostree=OSTreeImport.NO, function=diff_live_rootfs_img_tree),
     Differ("live-rootfs-img", "Diff live-rootfs.img content",
-           needs_ostree=False, function=diff_live_rootfs_img),
+           needs_ostree=OSTreeImport.NO, function=diff_live_rootfs_img),
     Differ("live-sysroot-ls", "Diff live '/root.[erofs|squash]fs' (embed into live-rootfs) listings",
-           needs_ostree=False, function=diff_live_sysroot_tree),
+           needs_ostree=OSTreeImport.NO, function=diff_live_sysroot_tree),
     Differ("live-sysroot", "Diff live '/root.[ero|squash]fs' (embed into live-rootfs) content",
-           needs_ostree=False, function=diff_live_sysroot),
+           needs_ostree=OSTreeImport.NO, function=diff_live_sysroot),
 ]
 
 if __name__ == '__main__':

src/cmd-koji-upload

@@ -668,7 +668,7 @@ class Upload(_KojiBase):
         if self._manifest is not None:
             return self._manifest
 
-        now = datetime.datetime.utcnow()
+        now = datetime.datetime.now(datetime.UTC)
         stamp = now.strftime("%s")
 
         """

src/cmdlib.sh

@@ -1066,7 +1066,7 @@ workdir = '${workdir:-$(pwd)}'
 builds = Builds(workdir)
 builddir = builds.get_build_dir('${buildid}')
 buildmeta = builds.get_build_meta('${buildid}')
-cmdlib.import_ostree_commit(workdir, builddir, buildmeta, ${extractjson})
+cmdlib.import_ostree_commit(workdir, builddir, buildmeta, extract_json=('${extractjson}' == '1'))
 ")
 }
 

src/cosalib/cmdlib.py

@@ -230,7 +230,7 @@ def rfc3339_time(t=None):
     :rtype: str
     """
     if t is None:
-        t = datetime.datetime.utcnow()
+        t = datetime.datetime.now(datetime.UTC)
     else:
         # if the need arises, we can convert to UTC, but let's just enforce
         # this doesn't slip by for now
@@ -283,7 +283,13 @@ def extract_image_json(workdir, commit):
 # a metal image, we may not have preserved that cache.
 #
 # Call this function to ensure that the ostree commit for a given build is in tmp/repo.
-def import_ostree_commit(workdir, buildpath, buildmeta, extract_json=1):
+#
+# Note also a user can request a partial import where just the commit object is
+# imported. This is a really lightweight way to get basic information like
+# version/commit metadata and enables things like rpm-ostree db diff.
+def import_ostree_commit(workdir, buildpath, buildmeta, extract_json=True, partial_import=False):
+    if extract_json and partial_import:
+        raise Exception("can't extract json from a partial import")
     tmpdir = os.path.join(workdir, 'tmp')
     with Lock(os.path.join(workdir, 'tmp/repo.import.lock'),
               lifetime=LOCK_DEFAULT_LIFETIME):
@@ -298,10 +304,27 @@ def import_ostree_commit(workdir, buildpath, buildmeta, extract_json=1):
         commitpartial = os.path.join(repo, f'state/{commit}.commitpartial')
         if (subprocess.call(['ostree', 'show', '--repo', repo, commit],
                             stdout=subprocess.DEVNULL,
-                            stderr=subprocess.DEVNULL) == 0
-                and not os.path.isfile(commitpartial)):
-            if extract_json == 1:
-                extract_image_json(workdir, commit)
+                            stderr=subprocess.DEVNULL) == 0):
+            if os.path.isfile(commitpartial):
+                if partial_import:
+                    # We have a partial commit (just the object), but the user only
+                    # requested a partial import so that's OK. We can return.
+                    return
+            else:
+                # We have the full commit. We can extract the json if requested and return.
+                if extract_json:
+                    extract_image_json(workdir, commit)
+                return
+
+        # If the user only requested a partial import then we'll just "import" the
+        # commit object itself into the repo.
+        if partial_import:
+            print(f"Importing {commit} object (partial import)")
+            commitobject = os.path.join(buildpath, 'ostree-commit-object')
+            commitpath = os.path.join(repo, f'objects/{commit[:2]}/{commit[2:]}.commit')
+            os.makedirs(os.path.dirname(commitpath), exist_ok=True)
+            shutil.copy(commitobject, commitpath)
+            open(commitpartial, 'w').close()
             return
 
         print(f"Extracting {commit}")
@@ -327,7 +350,7 @@ def import_ostree_commit(workdir, buildpath, buildmeta, extract_json=1):
                 subprocess.check_call(['ostree', f'--repo={repo}', 'pull-local', tmpd, buildmeta['buildid']])
 
         # Also extract image.json since it's commonly needed by image builds
-        if extract_json == 1:
+        if extract_json:
             extract_image_json(workdir, commit)