* [gentoo-commits] proj/tinderbox-cluster:master commit in: /, patches/
@ 2022-08-22 22:35 Magnus Granberg
0 siblings, 0 replies; only message in thread
From: Magnus Granberg @ 2022-08-22 22:35 UTC (permalink / raw
To: gentoo-commits
commit: b635f1ab52fa63964ff9fc01f67638810c2fd2bf
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 22 22:34:57 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Aug 22 22:34:57 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=b635f1ab
Add buildbot gitlab patch
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
master.cfg | 21 +++--
patches/bb-gitlab.patch | 212 +++++++++++++++++++++++++++++++++++++++++++++
patches/secretstring.patch | 74 ----------------
3 files changed, 225 insertions(+), 82 deletions(-)
diff --git a/master.cfg b/master.cfg
index af50ec2..dbdee3f 100644
--- a/master.cfg
+++ b/master.cfg
@@ -12,11 +12,13 @@ worker_data = [
{'uuid' : 'updatedb_2', 'password' : '', 'type' : 'local', 'enable' : True, },
{'uuid' : 'updatedb_3', 'password' : '', 'type' : 'local', 'enable' : True, },
{'uuid' : 'updatedb_4', 'password' : '', 'type' : 'local', 'enable' : True, },
- {'uuid' : 'c89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'password' : 'test', 'type' : 'log', 'enable' : True, },
- {'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'password' : 'test', 'type' : 'docker', 'enable' : True, },
- {'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfb', 'password' : 'test', 'type' : 'docker', 'enable' : True, },
- {'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfd', 'password' : 'test', 'type' : 'docker', 'enable' : True, },
- {'uuid' : 'c89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'password' : 'test', 'type' : 'node', 'enable' : True, },
+ {'uuid' : 'b89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'password' : 'X', 'type' : 'log', 'enable' : True, },
+ {'uuid' : 'b89c2c1a-46e0-4ded-81dd-c51afeb7fcfb', 'password' : 'X', 'type' : 'log', 'enable' : True, },
+ {'uuid' : 'b89c2c1a-46e0-4ded-81dd-c51afeb7fcfc', 'password' : 'X', 'type' : 'log', 'enable' : True, },
+ {'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'password' : 'X', 'type' : 'docker', 'enable' : True, },
+ {'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfb', 'password' : 'X', 'type' : 'docker', 'enable' : True, },
+ {'uuid' : 'a89c2c1a-46e0-4ded-81dd-c51afeb7fcfd', 'password' : 'X', 'type' : 'docker', 'enable' : True, },
+ {'uuid' : 'c89c2c1a-46e0-4ded-81dd-c51afeb7fcfa', 'password' : 'X', 'type' : 'node', 'enable' : True, },
]
# This is the dictionary that the buildmaster pays attention to. We also use
@@ -27,7 +29,8 @@ c['buildbotNetUsageData'] = None
####### SECRETS
-c['secretsProviders'] = [secrets.SecretInAFile(dirname="/var/lib/buildmaster/gentoo-ci-cloud/secrets/")]
+c['secretsProviders'] = []
+c['secretsProviders'].append(secrets.SecretInAFile(dirname="/var/lib/buildmaster/gentoo-ci-cloud/secrets"))
####### WORKERS
@@ -88,7 +91,7 @@ c['titleURL'] = "https://gentoo-ci.gentoo.org"
# the 'www' entry below, but with an externally-visible host name which the
# buildbot cannot figure out without some help.
-c['buildbotURL'] = "http://90.231.13.235:8010/"
+c['buildbotURL'] = "http://0.0.0.0:8010/"
# minimalistic config to activate new web UI
c['www'] = dict(
@@ -97,6 +100,8 @@ c['www'] = dict(
change_hook_dialects={
'gitlab' : {
'secret': util.Secret("WWWHookGitlabToken"),
+ 'token': util.Secret("gitlabToken"),
+ 'baseUrl': 'https://gitlab.gentoo.org',
},
},
)
@@ -116,4 +121,4 @@ c['www']['ui_default_config'] = {
# It's easy to start with sqlite, but it's recommended to switch to a dedicated
# database, such as PostgreSQL or MySQL, for use in production environments.
# http://docs.buildbot.net/current/manual/configuration/global.html#database-specification
-c['db_url'] = "postgresql://buildbot:fooo@192.168.1.9/buildbot"
+c['db_url'] = "postgresql://buildbot:X@192.0.0.0/buildbot"
diff --git a/patches/bb-gitlab.patch b/patches/bb-gitlab.patch
new file mode 100644
index 0000000..f57a2ac
--- /dev/null
+++ b/patches/bb-gitlab.patch
@@ -0,0 +1,212 @@
+--- a/buildbot/www/hooks/gitlab.py 2022-03-06 15:10:44.000000000 +0100
++++ b/buildbot/www/hooks/gitlab.py 2022-08-07 22:22:16.570498909 +0200
+@@ -19,6 +19,8 @@ import re
+
+ from dateutil.parser import parse as dateparse
+
++import gitlab as python_gitlab
++
+ from twisted.internet.defer import inlineCallbacks
+ from twisted.python import log
+
+@@ -26,6 +28,7 @@ from buildbot.process.properties import
+ from buildbot.util import bytes2unicode
+ from buildbot.www.hooks.base import BaseHookHandler
+
++_HOSTED_BASE_URL = 'https://gitlab.com'
+ _HEADER_EVENT = b'X-Gitlab-Event'
+ _HEADER_GITLAB_TOKEN = b'X-Gitlab-Token'
+
+@@ -94,6 +97,35 @@ class GitLabHandler(BaseHookHandler):
+
+ return changes
+
++ #@inlineCallbacks
++ def _configGitlabRest(self, token, baseURL=None):
++ if baseURL is None:
++ baseURL = _HOSTED_BASE_URL
++ if baseURL.endswith('/'):
++ baseURL = baseURL[:-1]
++ return python_gitlab.Gitlab(url=baseURL, private_token=token)
++
++ #@inlineCallbacks
++ def _getFiles(self, attrs):
++ """
++ get the files changes
++ """
++ files = []
++ project = self.gl.projects.get(attrs['target']['id'])
++ mr = project.mergerequests.get(attrs['iid'])
++ changes = mr.changes()
++ if isinstance(changes['changes'], list):
++ for change in changes['changes']:
++ print(change['old_path'])
++ print(change['new_path'])
++ print(change['new_file'])
++ print(change['renamed_file'])
++ print(change['deleted_file'])
++ if change['old_path'] == change['new_path']:
++ files.append(change['old_path'])
++ print(f"Files: {files}")
++ return files
++
+ def _process_merge_request_change(self, payload, event, codebase=None):
+ """
+ Consumes the merge_request JSON as a python object and turn it into a buildbot change.
+@@ -126,7 +158,7 @@ class GitLabHandler(BaseHookHandler):
+
+ changes = [{
+ 'author': f"{commit['author']['name']} <{commit['author']['email']}>",
+- 'files': [], # @todo use rest API
++ 'files' : self._getFiles(attrs),
+ 'comments': f"MR#{attrs['iid']}: {attrs['title']}\n\n{attrs['description']}",
+ 'revision': commit['id'],
+ 'when_timestamp': when_timestamp,
+@@ -151,6 +183,65 @@ class GitLabHandler(BaseHookHandler):
+ changes[0]['codebase'] = codebase
+ return changes
+
++ def _process_note_addition_to_merge_request(self, payload, event, codebase=None):
++ """
++ Consumes a note event JSON as a python object and turn it into a buildbot change.
++ :arguments:
++ payload
++ Python Object that represents the JSON sent by GitLab Service
++ Hook.
++ Comments in merge_requests are send as note events by the API
++ """
++ attrs = payload['object_attributes']
++
++ # handle only note events coming from merge_requests
++ # this can be direct comments or comments added to a changeset of the MR
++ #
++ # editing a comment does NOT lead to an event at all
++ if 'merge_request' not in payload:
++ log.msg(f"Found note event (id {attrs['id']}) without corresponding MR - ignore")
++ return []
++
++ # change handling is very similar to the method above, but
++ commit = payload['merge_request']['last_commit']
++ when_timestamp = dateparse(commit['timestamp'])
++ # @todo provide and document a way to choose between http and ssh url
++ repo_url = payload['merge_request']['target']['git_http_url']
++ # project name from http headers is empty for me, so get it from
++ # object_attributes/target/name
++ mr = payload['merge_request']
++ project = mr['target']['name']
++
++ log.msg(f"Found notes on MR#{mr['iid']}: {attrs['note']}")
++ changes = [{
++ 'author': f"{commit['author']['name']} <{commit['author']['email']}>",
++ 'files': [], # not provided by rest API
++ 'comments': f"MR#{mr['iid']}: {mr['title']}\n\n{mr['description']}",
++ 'revision': commit['id'],
++ 'when_timestamp': when_timestamp,
++ 'branch': mr['target_branch'],
++ 'repository': repo_url,
++ 'project': project,
++ 'category': event,
++ 'revlink': mr['url'],
++ 'properties': {
++ 'source_branch': mr['source_branch'],
++ 'source_project_id': mr['source_project_id'],
++ 'source_repository': mr['source']['git_http_url'],
++ 'source_git_ssh_url': mr['source']['git_ssh_url'],
++ 'target_branch': mr['target_branch'],
++ 'target_project_id': mr['target_project_id'],
++ 'target_repository': mr['target']['git_http_url'],
++ 'target_git_ssh_url': mr['target']['git_ssh_url'],
++ 'event': event,
++ 'comments': attrs['note'],
++ },
++ }]
++ if codebase is not None:
++ changes[0]['codebase'] = codebase
++ return changes
++
++
+ @inlineCallbacks
+ def getChanges(self, request):
+ """
+@@ -160,17 +251,21 @@ class GitLabHandler(BaseHookHandler):
+ request
+ the http request object
+ """
++ p = Properties()
++ p.master = self.master
+ expected_secret = isinstance(self.options, dict) and self.options.get('secret')
+ if expected_secret:
+ received_secret = request.getHeader(_HEADER_GITLAB_TOKEN)
+ received_secret = bytes2unicode(received_secret)
+-
+- p = Properties()
+- p.master = self.master
+ expected_secret_value = yield p.render(expected_secret)
+
+ if received_secret != expected_secret_value:
+ raise ValueError("Invalid secret")
++
++ baseUrl_value = isinstance(self.options, dict) and self.options.get('baseUrl')
++ expected_token = isinstance(self.options, dict) and self.options.get('token')
++ expected_token_value = yield p.render(expected_token)
++
+ try:
+ content = request.content.read()
+ payload = json.loads(bytes2unicode(content))
+@@ -189,7 +284,11 @@ class GitLabHandler(BaseHookHandler):
+ repo_url = payload['repository']['url']
+ changes = self._process_change(
+ payload, user, repo, repo_url, event_type, codebase=codebase)
++ elif event_type == 'note':
++ changes = self._process_note_addition_to_merge_request(
++ payload, event_type, codebase=codebase)
+ elif event_type == 'merge_request':
++ self.gl = self._configGitlabRest(expected_token_value, baseURL=baseUrl_value)
+ changes = self._process_merge_request_change(
+ payload, event_type, codebase=codebase)
+ else:
+--- a/buildbot/reporters/gitlab.py 2022-03-06 15:10:44.000000000 +0100
++++ b/buildbot/reporters/gitlab.py 2022-08-23 00:11:55.996195542 +0200
+@@ -95,7 +95,7 @@ class GitLabStatusPush(ReporterBase):
+ :param branch: Branch name to create the status for.
+ :param sha: Full sha to create the status for.
+ :param state: one of the following 'pending', 'success', 'failed'
+- or 'cancelled'.
++ or 'canceled'.
+ :param target_url: Target url to associate with this status.
+ :param description: Short description of the status.
+ :param context: Context of the result
+@@ -155,10 +155,12 @@ class GitLabStatusPush(ReporterBase):
+ SKIPPED: 'success',
+ EXCEPTION: 'failed',
+ RETRY: 'pending',
+- CANCELLED: 'cancelled'
++ CANCELLED: 'canceled'
+ }.get(build['results'], 'failed')
+- else:
++ elif build.get('started_at'):
+ state = 'running'
++ else:
++ state = 'pending'
+
+ context = yield props.render(self.context)
+
+@@ -167,8 +169,8 @@ class GitLabStatusPush(ReporterBase):
+ # FIXME: probably only want to report status for the last commit in the changeset
+ for sourcestamp in sourcestamps:
+ sha = sourcestamp['revision']
+- if 'source_project_id' in props:
+- proj_id = props['source_project_id']
++ if 'target_project_id' in props:
++ proj_id = props['target_project_id']
+ else:
+ proj_id = yield self.getProjectId(sourcestamp)
+ if proj_id is None:
+@@ -189,6 +191,7 @@ class GitLabStatusPush(ReporterBase):
+ description=description
+ )
+ if res.code not in (200, 201, 204):
++ log.msg(res.code)
+ message = yield res.json()
+ message = message.get('message', 'unspecified error')
+ log.msg(
diff --git a/patches/secretstring.patch b/patches/secretstring.patch
deleted file mode 100644
index 4e0833d..0000000
--- a/patches/secretstring.patch
+++ /dev/null
@@ -1,74 +0,0 @@
-diff --git a/master/buildbot/process/buildstep.py b/master/buildbot/process/buildstep.py
-index 7ac18b086..82ff7ce07 100644
---- a/master/buildbot/process/buildstep.py
-+++ b/master/buildbot/process/buildstep.py
-@@ -850,6 +850,7 @@ class ShellMixin:
- 'sigtermTime',
- 'initialStdin',
- 'decodeRC',
-+ 'SecretString',
- ]
- renderables = _shellMixinArgs
-
-diff --git a/master/buildbot/process/remotecommand.py b/master/buildbot/process/remotecommand.py
-index 90ec4c44d..fb3099b28 100644
---- a/master/buildbot/process/remotecommand.py
-+++ b/master/buildbot/process/remotecommand.py
-@@ -44,7 +44,7 @@ class RemoteCommand(base.RemoteCommandImpl):
-
- def __init__(self, remote_command, args, ignore_updates=False,
- collectStdout=False, collectStderr=False, decodeRC=None,
-- stdioLogName='stdio'):
-+ stdioLogName='stdio', SecretString=False):
- if decodeRC is None:
- decodeRC = {0: SUCCESS}
- self.logs = {}
-@@ -70,6 +70,7 @@ class RemoteCommand(base.RemoteCommandImpl):
- self.commandID = None
- self.deferred = None
- self.interrupted = False
-+ self.SecretString = SecretString
- # a lock to make sure that only one log-handling method runs at a time.
- # This is really only a problem with old-style steps, which do not
- # wait for the Deferred from one method before invoking the next.
-@@ -275,6 +276,8 @@ class RemoteCommand(base.RemoteCommandImpl):
- def cleanup(data):
- if self.step is None:
- return data
-+ if self.SecretString and isinstance(self.SecretString, list) and len(self.SecretString) == 2:
-+ data = data.replace(self.SecretString[0], '<' + self.SecretString[1] + '>')
- return self.step.build.properties.cleanupTextFromSecrets(data)
-
- if self.debug:
-@@ -358,7 +361,8 @@ class RemoteShellCommand(RemoteCommand):
- collectStdout=False, collectStderr=False,
- interruptSignal=None,
- initialStdin=None, decodeRC=None,
-- stdioLogName='stdio'):
-+ stdioLogName='stdio',
-+ SecretString=False):
- if logfiles is None:
- logfiles = {}
- if decodeRC is None:
-@@ -398,7 +402,8 @@ class RemoteShellCommand(RemoteCommand):
- super().__init__("shell", args, collectStdout=collectStdout,
- collectStderr=collectStderr,
- decodeRC=decodeRC,
-- stdioLogName=stdioLogName)
-+ stdioLogName=stdioLogName,
-+ SecretString=SecretString)
-
- def _start(self):
- if self.args['usePTY'] is None:
-diff --git a/master/buildbot/steps/shell.py b/master/buildbot/steps/shell.py
-index d21ca7b9b..cf56a2802 100644
---- a/master/buildbot/steps/shell.py
-+++ b/master/buildbot/steps/shell.py
-@@ -183,6 +183,7 @@ class ShellCommand(buildstep.ShellMixin, buildstep.BuildStep):
- 'decodeRC',
- 'stdioLogName',
- 'workdir',
-+ 'SecretString',
- ] + buildstep.BuildStep.parms
-
- invalid_args = []
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2022-08-22 22:35 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2022-08-22 22:35 [gentoo-commits] proj/tinderbox-cluster:master commit in: /, patches/ Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox