Skip to content

Commit b06ff5d

Browse files
cottsaycottsay
committed
Add RPM support for sync-to-testing job
1 parent 9f2ae6f commit b06ff5d

File tree

3 files changed

+289
-6
lines changed

3 files changed

+289
-6
lines changed

‎ros_buildfarm/release_job.py

+12-6
Original file line numberDiff line numberDiff line change
@@ -691,7 +691,7 @@ def _get_binarydeb_job_config(
691691
for var, value in sorted(build_file.build_environment_variables.items())]
692692

693693
sync_to_testing_job_name = [get_sync_packages_to_testing_job_name(
694-
rosdistro_name, os_code_name, arch)]
694+
rosdistro_name, os_name, os_code_name, arch)]
695695

696696
maintainer_emails = _get_maintainer_emails(cached_pkgs, pkg_name) \
697697
if build_file.notify_maintainers \
@@ -806,7 +806,7 @@ def configure_sync_packages_to_testing_job(
806806
jenkins = connect(config.jenkins_url)
807807

808808
job_name = get_sync_packages_to_testing_job_name(
809-
rosdistro_name, os_code_name, arch)
809+
rosdistro_name, os_name, os_code_name, arch)
810810
job_config = _get_sync_packages_to_testing_job_config(
811811
config_url, rosdistro_name, release_build_name, os_name, os_code_name,
812812
arch, config, build_file)
@@ -819,16 +819,18 @@ def configure_sync_packages_to_testing_job(
819819

820820

821821
def get_sync_packages_to_testing_job_name(
822-
rosdistro_name, os_code_name, arch):
822+
rosdistro_name, os_name, os_code_name, arch):
823823
view_name = get_release_job_prefix(rosdistro_name)
824-
return '%s_sync-packages-to-testing_%s_%s' % \
825-
(view_name, os_code_name, arch)
824+
return '%s_sync-packages-to-testing%s_%s_%s' % (
825+
view_name, '' if package_format_mapping[os_name] == 'deb' else '_' + os_name,
826+
os_code_name, arch)
826827

827828

828829
def _get_sync_packages_to_testing_job_config(
829830
config_url, rosdistro_name, release_build_name, os_name, os_code_name,
830831
arch, config, build_file):
831-
template_name = 'release/deb/sync_packages_to_testing_job.xml.em'
832+
package_format = package_format_mapping[os_name]
833+
template_name = 'release/%s/sync_packages_to_testing_job.xml.em' % package_format
832834

833835
repository_args, script_generating_key_files = \
834836
get_repositories_and_script_generating_key_files(build_file=build_file)
@@ -846,7 +848,11 @@ def _get_sync_packages_to_testing_job_config(
846848
'arch': arch,
847849
'repository_args': repository_args,
848850

851+
'import_package_job_name': get_import_package_job_name(
852+
rosdistro_name, package_format),
853+
849854
'notify_emails': build_file.notify_emails,
855+
'credential_id': build_file.upload_credential_id,
850856
}
851857
job_config = expand_template(template_name, job_data)
852858
return job_config

‎ros_buildfarm/templates/release/rpm/sync_packages_to_testing_job.xml.em

+177
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
<project>
2+
<actions/>
3+
<description>Generated at @ESCAPE(now_str) from template '@ESCAPE(template_name)'</description>
4+
<keepDependencies>false</keepDependencies>
5+
<properties>
6+
@(SNIPPET(
7+
'property_log-rotator',
8+
days_to_keep=100,
9+
num_to_keep=100,
10+
))@
11+
@(SNIPPET(
12+
'property_job-priority',
13+
priority=-1,
14+
))@
15+
@(SNIPPET(
16+
'property_requeue-job',
17+
))@
18+
</properties>
19+
@(SNIPPET(
20+
'scm_git',
21+
url=ros_buildfarm_repository.url,
22+
branch_name=ros_buildfarm_repository.version or 'master',
23+
relative_target_dir='ros_buildfarm',
24+
refspec=None,
25+
))@
26+
<scmCheckoutRetryCount>2</scmCheckoutRetryCount>
27+
<assignedNode>building_repository</assignedNode>
28+
<canRoam>false</canRoam>
29+
<disabled>false</disabled>
30+
<blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
31+
<blockBuildWhenUpstreamBuilding>true</blockBuildWhenUpstreamBuilding>
32+
<triggers/>
33+
<concurrentBuild>false</concurrentBuild>
34+
<builders>
35+
@(SNIPPET(
36+
'builder_shell_docker-info',
37+
))@
38+
@(SNIPPET(
39+
'builder_shell',
40+
script='\n'.join([
41+
'# generate key files',
42+
'echo "# BEGIN SECTION: Write PGP repository keys"',
43+
'mkdir -p $WORKSPACE/keys',
44+
'rm -fr $WORKSPACE/keys/*',
45+
'echo "-----BEGIN PGP PUBLIC KEY BLOCK-----\nVersion: GnuPG v1\n\n\nmQINBFzvJpYBEADY8l1YvO7iYW5gUESyzsTGnMvVUmlV3XarBaJz9bGRmgPXh7jc\nVFrQhE0L/HV7LOfoLI9H2GWYyHBqN5ERBlcA8XxG3ZvX7t9nAZPQT2Xxe3GT3tro\nu5oCR+SyHN9xPnUwDuqUSvJ2eqMYb9B/Hph3OmtjG30jSNq9kOF5bBTk1hOTGPH4\nK/AY0jzT6OpHfXU6ytlFsI47ZKsnTUhipGsKucQ1CXlyirndZ3V3k70YaooZ55rG\naIoAWlx2H0J7sAHmqS29N9jV9mo135d+d+TdLBXI0PXtiHzE9IPaX+ctdSUrPnp+\nTwR99lxglpIG6hLuvOMAaxiqFBB/Jf3XJ8OBakfS6nHrWH2WqQxRbiITl0irkQoz\npwNEF2Bv0+Jvs1UFEdVGz5a8xexQHst/RmKrtHLct3iOCvBNqoAQRbvWvBhPjO/p\nV5cYeUljZ5wpHyFkaEViClaVWqa6PIsyLqmyjsruPCWlURLsQoQxABcL8bwxX7UT\nhM6CtH6tGlYZ85RIzRifIm2oudzV5l+8oRgFr9yVcwyOFT6JCioqkwldW52P1pk/\n/SnuexC6LYqqDuHUs5NnokzzpfS6QaWfTY5P5tz4KHJfsjDIktly3mKVfY0fSPVV\nokdGpcUzvz2hq1fqjxB6MlB/1vtk0bImfcsoxBmF7H+4E9ZN1sX/tSb0KQARAQAB\ntCZPcGVuIFJvYm90aWNzIDxpbmZvQG9zcmZvdW5kYXRpb24ub3JnPokCVAQTAQoA\nPhYhBMHPbjHmut6IaLFytPQu1vurF8ZUBQJc7yaWAhsDBQkDwmcABQsJCAcCBhUK\nCQgLAgQWAgMBAh4BAheAAAoJEPQu1vurF8ZUkhIP/RbZY1ErvCEUy8iLJm9aSpLQ\nnDZl5xILOxyZlzpg+Ml5bb0EkQDr92foCgcvLeANKARNCaGLyNIWkuyDovPV0xZJ\nrEy0kgBrDNb3++NmdI/+GA92pkedMXXioQvqdsxUagXAIB/sNGByJEhs37F05AnF\nvZbjUhceq3xTlvAMcrBWrgB4NwBivZY6IgLvl/CRQpVYwANShIQdbvHvZSxRonWh\nNXr6v/Wcf8rsp7g2VqJ2N2AcWT84aa9BLQ3Oe/SgrNx4QEhA1y7rc3oaqPVu5ZXO\nK+4O14JrpbEZ3Xs9YEjrcOuEDEpYktA8qqUDTdFyZrxb9S6BquUKrA6jZgT913kj\nJ4e7YAZobC4rH0w4u0PrqDgYOkXA9Mo7L601/7ZaDJob80UcK+Z12ZSw73IgBix6\nDiJVfXuWkk5PM2zsFn6UOQXUNlZlDAOj5NC01V0fJ8P0v6GO9YOSSQx0j5UtkUbR\nfp/4W7uCPFvwAatWEHJhlM3sQNiMNStJFegr56xQu1a/cbJH7GdbseMhG/f0BaKQ\nqXCI3ffB5y5AOLc9Hw7PYiTFQsuY1ePRhE+J9mejgWRZxkjAH/FlAubqXkDgterC\nh+sLkzGf+my2IbsMCuc+3aeNMJ5Ej/vlXefCH/MpPWAHCqpQhe2DET/jRSaM53US\nAHNx8kw4MPUkxExgI7Sd\n=4Ofr\n-----END PGP PUBLIC KEY BLOCK-----" > $WORKSPACE/keys/0.key',
46+
'echo "# END SECTION"',
47+
]),
48+
))@
49+
@(SNIPPET(
50+
'builder_shell',
51+
script='\n'.join([
52+
'rm -fr $WORKSPACE/docker_check_sync_criteria',
53+
'mkdir -p $WORKSPACE/docker_check_sync_criteria',
54+
'',
55+
'# monitor all subprocesses and enforce termination',
56+
'python3 -u $WORKSPACE/ros_buildfarm/scripts/subprocess_reaper.py $$ --cid-file $WORKSPACE/docker_check_sync_criteria/docker.cid > $WORKSPACE/docker_check_sync_criteria/subprocess_reaper.log 2>&1 &',
57+
'# sleep to give python time to startup',
58+
'sleep 1',
59+
'',
60+
'# generate Dockerfile, build and run it',
61+
'# checking the sync criteria',
62+
'echo "# BEGIN SECTION: Generate Dockerfile - check sync condition"',
63+
'export TZ="%s"' % timezone,
64+
'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
65+
'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/run_check_sync_criteria_job.py' +
66+
' ' + config_url +
67+
' ' + rosdistro_name +
68+
' ' + release_build_name +
69+
' ' + os_name +
70+
' ' + os_code_name +
71+
' ' + arch +
72+
' --distribution-repository-urls http://repositories.ros.org/ubuntu/main' +
73+
' --distribution-repository-key-files $WORKSPACE/keys/0.key' +
74+
' --cache-dir /tmp/package_repo_cache' +
75+
' --dockerfile-dir $WORKSPACE/docker_check_sync_criteria',
76+
'echo "# END SECTION"',
77+
'',
78+
'echo "# BEGIN SECTION: Build Dockerfile - check sync condition"',
79+
'cd $WORKSPACE/docker_check_sync_criteria',
80+
'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
81+
'docker build --force-rm -t check_sync_condition .',
82+
'echo "# END SECTION"',
83+
'',
84+
'echo "# BEGIN SECTION: Run Dockerfile - check sync condition"',
85+
'rm -fr $WORKSPACE/package_repo_cache',
86+
'mkdir -p $WORKSPACE/package_repo_cache',
87+
'docker run' +
88+
' --rm ' +
89+
' --cidfile=$WORKSPACE/docker_check_sync_criteria/docker.cid' +
90+
' -v $WORKSPACE/ros_buildfarm:/tmp/ros_buildfarm:ro' +
91+
' -v $WORKSPACE/package_repo_cache:/tmp/package_repo_cache' +
92+
' check_sync_condition',
93+
'echo "# END SECTION"',
94+
]),
95+
))@
96+
@(SNIPPET(
97+
'builder_shell',
98+
script='\n'.join([
99+
'echo "# BEGIN SECTION: determine source packages to sync"',
100+
'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
101+
'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/list_packages.py' +
102+
' --pulp-base-url http://repo:24817' +
103+
' --pulp-distribution-name ros-building-%s-%s-SRPMS' % (os_name, os_code_name) +
104+
' --package-name-expression "^ros-%s-.*"' % rosdistro_name +
105+
' --pulp-resource-record $WORKSPACE/ros-building-source.txt',
106+
'echo "# END SECTION"',
107+
'',
108+
'echo "# BEGIN SECTION: determine binary packages to sync"',
109+
'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
110+
'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/list_packages.py' +
111+
' --pulp-base-url http://repo:24817' +
112+
' --pulp-distribution-name ros-building-%s-%s-%s' % (os_name, os_code_name, arch) +
113+
' --package-name-expression "^ros-%s-.*"' % rosdistro_name +
114+
' --pulp-resource-record $WORKSPACE/ros-building-binary.txt',
115+
'echo "# END SECTION"',
116+
'',
117+
'echo "# BEGIN SECTION: determine binary debug packages to testing repo"',
118+
'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
119+
'python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/list_packages.py' +
120+
' --pulp-base-url http://repo:24817' +
121+
' --pulp-distribution-name ros-building-%s-%s-%s-debug' % (os_name, os_code_name, arch) +
122+
' --package-name-expression "^ros-%s-.*"' % rosdistro_name +
123+
' --pulp-resource-record $WORKSPACE/ros-building-binary-debug.txt',
124+
'echo "# END SECTION"',
125+
]),
126+
))@
127+
@(SNIPPET(
128+
'builder_shell',
129+
script='\n'.join([
130+
'echo "# BEGIN SECTION: sync source packages to testing repo"',
131+
'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
132+
'sed "s/^PULP_RESOURCES=//" $WORKSPACE/ros-building-source.txt | xargs' +
133+
' python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/import_package.py' +
134+
' --pulp-base-url http://repo:24817' +
135+
' --pulp-distribution-name ros-testing-%s-%s-SRPMS' % (os_name, os_code_name) +
136+
' --invalidate-expression "^ros-%s-.*"' % rosdistro_name,
137+
'echo "# END SECTION"',
138+
'',
139+
'echo "# BEGIN SECTION: sync binary packages to testing repo"',
140+
'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
141+
'sed "s/^PULP_RESOURCES=//" $WORKSPACE/ros-building-binary.txt | xargs' +
142+
' python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/import_package.py' +
143+
' --pulp-base-url http://repo:24817' +
144+
' --pulp-distribution-name ros-testing-%s-%s-%s' % (os_name, os_code_name, arch) +
145+
' --invalidate-expression "^ros-%s-.*"' % rosdistro_name,
146+
'echo "# END SECTION"',
147+
'',
148+
'echo "# BEGIN SECTION: sync binary debug packages to testing repo"',
149+
'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
150+
'sed "s/^PULP_RESOURCES=//" $WORKSPACE/ros-building-binary-debug.txt | xargs' +
151+
' python3 -u $WORKSPACE/ros_buildfarm/scripts/release/rpm/import_package.py' +
152+
' --pulp-base-url http://repo:24817' +
153+
' --pulp-distribution-name ros-testing-%s-%s-%s-debug' % (os_name, os_code_name, arch) +
154+
' --invalidate-expression "^ros-%s-.*"' % rosdistro_name,
155+
'echo "# END SECTION"',
156+
]),
157+
))@
158+
</builders>
159+
<publishers>
160+
@(SNIPPET(
161+
'publisher_mailer',
162+
recipients=notify_emails,
163+
dynamic_recipients=[],
164+
send_to_individuals=False,
165+
))@
166+
</publishers>
167+
<buildWrappers>
168+
@(SNIPPET(
169+
'credentials_binding_plugin',
170+
credential_id=credential_id,
171+
env_var_prefix='PULP',
172+
))@
173+
@(SNIPPET(
174+
'build-wrapper_timestamper',
175+
))@
176+
</buildWrappers>
177+
</project>

‎scripts/release/rpm/list_packages.py

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
#!/usr/bin/env python3
2+
3+
# Copyright 2020 Open Source Robotics Foundation, Inc.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import argparse
18+
import re
19+
import sys
20+
21+
from pulpcore.client import pulp_rpm
22+
from ros_buildfarm.argument import add_argument_pulp_base_url
23+
from ros_buildfarm.argument import add_argument_pulp_distribution_name
24+
from ros_buildfarm.argument import add_argument_pulp_password
25+
from ros_buildfarm.argument import add_argument_pulp_resource_record
26+
from ros_buildfarm.argument import add_argument_pulp_task_timeout
27+
from ros_buildfarm.argument import add_argument_pulp_username
28+
from ros_buildfarm.common import Scope
29+
30+
31+
def main(argv=sys.argv[1:]):
32+
parser = argparse.ArgumentParser(
33+
description='List the pulp IDs of packages provided at a given pulp distribution endpoint')
34+
add_argument_pulp_base_url(parser)
35+
add_argument_pulp_distribution_name(parser)
36+
add_argument_pulp_password(parser)
37+
add_argument_pulp_resource_record(parser)
38+
add_argument_pulp_task_timeout(parser)
39+
add_argument_pulp_username(parser)
40+
parser.add_argument(
41+
'--package-name-expression', default=None,
42+
help='Expression to match against packages in the repository')
43+
44+
args = parser.parse_args(argv)
45+
46+
pulp_config = pulp_rpm.Configuration(
47+
args.pulp_base_url, username=args.pulp_username,
48+
password=args.pulp_password)
49+
50+
# https://pulp.plan.io/issues/5932
51+
pulp_config.safe_chars_for_path_param = '/'
52+
53+
pulp_rpm_client = pulp_rpm.ApiClient(pulp_config)
54+
pulp_packages_api = pulp_rpm.ContentPackagesApi(pulp_rpm_client)
55+
pulp_distributions_api = pulp_rpm.DistributionsRpmApi(pulp_rpm_client)
56+
pulp_publications_api = pulp_rpm.PublicationsRpmApi(pulp_rpm_client)
57+
58+
with Scope('SUBSECTION', 'list packages in the distribution'):
59+
distribution = pulp_distributions_api.list(
60+
name=args.pulp_distribution_name).results[0]
61+
print('Pulp Distribution: ' + pulp_config.host + distribution.pulp_href)
62+
63+
publication = pulp_publications_api.read(distribution.publication)
64+
print('Pulp Publication: %s%s' % (pulp_config.host, publication.pulp_href))
65+
print('Pulp Repository: %s%s' % (pulp_config.host, publication.repository))
66+
print('Pulp Repository Version: %s%s' % (pulp_config.host, publication.repository_version))
67+
68+
packages_in_version_page = pulp_packages_api.list(
69+
repository_version=publication.repository_version, limit=200)
70+
packages_in_version = list(packages_in_version_page.results)
71+
while packages_in_version_page.next:
72+
packages_in_version_page = pulp_packages_api.list(
73+
repository_version=publication.repository_version, limit=200,
74+
offset=len(packages_in_version))
75+
packages_in_version += packages_in_version_page.results
76+
77+
print('Total packages in repository: %d' % len(packages_in_version))
78+
79+
if args.package_name_expression:
80+
compiled_expression = re.compile(args.package_name_expression)
81+
packages_in_version = [
82+
pkg for pkg in packages_in_version if compiled_expression.match(pkg.name)]
83+
84+
print('Found %d packages:' % len(packages_in_version))
85+
for pkg in packages_in_version:
86+
print('- %s-%s%s-%s' % (
87+
pkg.name,
88+
(pkg.epoch + ':') if pkg.epoch != '0' else '',
89+
pkg.version,
90+
pkg.release))
91+
92+
if args.pulp_resource_record:
93+
print("Saving pulp resource record to '%s'." % args.pulp_resource_record)
94+
with open(args.pulp_resource_record, 'w') as resource_record:
95+
resource_record.write('PULP_RESOURCES=%s\n' % (
96+
' '.join([pkg.pulp_href for pkg in packages_in_version])))
97+
98+
99+
if __name__ == '__main__':
100+
sys.exit(main())

0 commit comments

Comments
 (0)