* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
@ 2021-04-10 1:03 Magnus Granberg
0 siblings, 0 replies; 5+ messages in thread
From: Magnus Granberg @ 2021-04-10 1:03 UTC (permalink / raw
To: gentoo-commits
commit: 550ae33d23ae93d7d779922a0b8e07710458c1a7
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 10 01:02:38 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Apr 10 01:02:38 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=550ae33d
Add support for Failed to XX in emerge build
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/packages.py | 4 +-
buildbot_gentoo_ci/db/versions.py | 3 +-
buildbot_gentoo_ci/steps/builders.py | 90 ++++++++++++++++++++++++++++--------
buildbot_gentoo_ci/steps/logs.py | 7 ++-
buildbot_gentoo_ci/steps/package.py | 4 +-
buildbot_gentoo_ci/steps/version.py | 2 +-
6 files changed, 85 insertions(+), 25 deletions(-)
diff --git a/buildbot_gentoo_ci/db/packages.py b/buildbot_gentoo_ci/db/packages.py
index 13cc267..a22b9d8 100644
--- a/buildbot_gentoo_ci/db/packages.py
+++ b/buildbot_gentoo_ci/db/packages.py
@@ -27,11 +27,13 @@ from buildbot.db import base
class PackagesConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
- def getPackageByName(self, name):
+ def getPackageByName(self, name, c_uuid, repo_uuid):
def thd(conn):
tbl = self.db.model.packages
q = tbl.select()
q = q.where(tbl.c.name == name)
+ q = q.where(tbl.c.category_uuid == c_uuid)
+ q = q.where(tbl.c.repository_uuid == repo_uuid)
res = conn.execute(q)
row = res.fetchone()
if not row:
diff --git a/buildbot_gentoo_ci/db/versions.py b/buildbot_gentoo_ci/db/versions.py
index 85bd105..768c1dc 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -28,11 +28,12 @@ from buildbot.util import datetime2epoch
class VersionsConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
- def getVersionByName(self, name):
+ def getVersionByName(self, name, p_uuid):
def thd(conn):
tbl = self.db.model.versions
q = tbl.select()
q = q.where(tbl.c.name == name)
+ q = q.where(tbl.c.package_uuid == p_uuid)
res = conn.execute(q)
row = res.fetchone()
if not row:
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index af3be9d..e51576f 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -4,7 +4,7 @@
import os
import re
-from portage.versions import catpkgsplit
+from portage.versions import catpkgsplit, cpv_getversion
from twisted.internet import defer
from twisted.python import log
@@ -19,10 +19,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
emerge_output['rc'] = rc
emerge_output['preserved_libs'] = False
emerge_output['change_use'] = False
+ emerge_output['failed'] = False
package_dict = {}
log_path_list = []
print(stderr)
- emerge_output['stderr'] = stderr
# split the lines
for line in stdout.split('\n'):
# package list
@@ -67,6 +67,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
# CPU_FLAGS_X86 list
package_dict[cpv] = subdict
if line.startswith('>>>'):
+ if line.startswith('>>> Failed to'):
+ emerge_output['failed'] = line.split(' ')[4][:-1]
+ if line.endswith('.log.gz') and emerge_output['failed']:
+ log_path_list.append(line.split(' ')[2])
#FIXME: Handling of >>> output
pass
if line.startswith('!!!'):
@@ -78,9 +82,10 @@ def PersOutputOfEmerge(rc, stdout, stderr):
log_path_list.append(line.split(' ')[4])
#FIXME: Handling of depclean output dict of packages that get removed or saved
emerge_output['package'] = package_dict
- emerge_output['log_paths'] = log_path_list
+
# split the lines
#FIXME: Handling of stderr output
+ stderr_line_list = []
for line in stderr.split('\n'):
if 'Change USE:' in line:
line_list = line.split(' ')
@@ -96,6 +101,16 @@ def PersOutputOfEmerge(rc, stdout, stderr):
else:
change_use_list = False
emerge_output['change_use'] = change_use_list
+ err_line_list = []
+ if line.startswith(' * '):
+ if line.endswith('.log.gz'):
+ log_path = line.split(' ')[3]
+ if log_path not in inlog_path_list:
+ log_path_list.append(log_path)
+ stderr_line_list.append(line)
+ emerge_output['stderr'] = stderr_line_list
+ emerge_output['log_paths'] = log_path_list
+
return {
'emerge_output' : emerge_output
}
@@ -484,6 +499,30 @@ class CheckEmergeLogs(BuildStep):
self.step = step
super().__init__(**kwargs)
self.descriptionSuffix = self.step
+ self.aftersteps_list = []
+ self.log_data = {}
+
+ @defer.inlineCallbacks
+ def getVersionData(self, cpv):
+ c = yield catpkgsplit(cpv)[0]
+ p = yield catpkgsplit(cpv)[1]
+ category_data = yield self.gentooci.db.categorys.getCategoryByName(c)
+ package_data = yield self.gentooci.db.packages.getPackageByName(p,
+ category_data['uuid'],
+ self.getProperty('repository_data')['uuid'])
+ if package_data is None:
+ return None
+ version = yield cpv_getversion(cpv)
+ version_data = yield self.gentooci.db.versions.getVersionByName(version, package_data['uuid'])
+ return version_data
+
+ @defer.inlineCallbacks
+ def getLogFile(self, cpv, log_dict):
+ masterdest = yield os.path.join(self.master.basedir, 'cpv_logs', log_dict[cpv]['full_logname'])
+ self.aftersteps_list.append(steps.FileUpload(
+ workersrc=log_dict[cpv]['log_path'],
+ masterdest=masterdest
+ ))
@defer.inlineCallbacks
def run(self):
@@ -495,7 +534,6 @@ class CheckEmergeLogs(BuildStep):
'emerge',
'-v'
]
- aftersteps_list = []
#FIXME: Prosees the logs and do stuff
# preserved-libs
@@ -533,13 +571,13 @@ class CheckEmergeLogs(BuildStep):
else:
change_use_list.append(use_flag)
change_use_string = separator2.join(change_use_list)
- aftersteps_list.append(
+ self.aftersteps_list.append(
steps.StringDownload(change_use_string + separator,
workerdest='zz_autouse' + str(self.getProperty('rerun')),
workdir='/etc/portage/package.use/')
)
# rerun
- aftersteps_list.append(RunEmerge(step='pre-build'))
+ self.aftersteps_list.append(RunEmerge(step='pre-build'))
self.setProperty('rerun', self.getProperty('rerun') + 1, 'rerun')
else:
# trigger parse_build_log with info about pre-build and it fail
@@ -564,31 +602,43 @@ class CheckEmergeLogs(BuildStep):
full_logname = full_logname
)
print(log_dict)
- # Find log for cpv that was requested or did faild
+ # Find log for cpv that was requested or did failed
if not log_dict == {}:
# requested cpv
- if self.getProperty('cpv') in log_dict:
- log_data = log_dict[self.getProperty('cpv')]
- masterdest = yield os.path.join(self.master.basedir, 'cpv_logs', log_data['full_logname'])
- aftersteps_list.append(steps.FileUpload(
- workersrc=log_data['log_path'],
- masterdest=masterdest
- ))
- aftersteps_list.append(steps.Trigger(
+ print(log_dict)
+ cpv = self.getProperty('cpv')
+ faild_cpv = emerge_output['failed']
+ if cpv in log_dict or faild_cpv in log_dict:
+ if cpv in log_dict:
+ self.log_data[cpv] = log_dict[cpv]
+ yield self.getLogFile(cpv, log_dict)
+ faild_version_data = False
+ if faild_cpv:
+ # failed and build requested cpv
+ if cpv == faild_cpv:
+ faild_version_data = self.getProperty("version_data")
+ else:
+ # failed but not build requested cpv
+ self.log_data[faild_cpv] = log_dict[faild_cpv]
+ yield self.getLogFile(faild_cpv, log_dict)
+ faild_version_data = yield self.getVersionData(faild_cpv)
+ self.aftersteps_list.append(steps.Trigger(
schedulerNames=['parse_build_log'],
waitForFinish=False,
updateSourceStamp=False,
set_properties={
'cpv' : self.getProperty("cpv"),
- 'faild_version_data' : self.getProperty('faild_version_data'),
+ 'faild_version_data' : faild_version_data,
'project_build_data' : self.getProperty('project_build_data'),
- 'log_build_data' : log_data,
+ 'log_build_data' : self.log_data,
'pkg_check_log_data' : self.getProperty("pkg_check_log_data"),
- 'repository_data' : self.getProperty('repository_data')
+ 'repository_data' : self.getProperty('repository_data'),
+ 'faild_cpv' : faild_cpv,
+ 'step' : self.step
}
))
- if not self.step is None and aftersteps_list != []:
- yield self.build.addStepsAfterCurrentStep(aftersteps_list)
+ if not self.step is None and self.aftersteps_list != []:
+ yield self.build.addStepsAfterCurrentStep(self.aftersteps_list)
return SUCCESS
class CheckDepcleanLogs(BuildStep):
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 767fdd4..4c3bf01 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -157,7 +157,12 @@ class ParserBuildLog(BuildStep):
# read it to a buffer
# make a dict of the buffer
# maby use mulitiprocces to speed up the search
- file_path = yield os.path.join(self.master.basedir, 'cpv_logs', self.getProperty('log_build_data')['full_logname'])
+ print(self.getProperty('log_build_data'))
+ if self.getProperty('faild_cpv'):
+ log_cpv = self.getProperty('log_build_data')[self.getProperty('faild_cpv')]
+ else:
+ log_cpv = self.getProperty('log_build_data')[self.getProperty('cpv')]
+ file_path = yield os.path.join(self.master.basedir, 'cpv_logs', log_cpv['full_logname'])
with io.TextIOWrapper(io.BufferedReader(gzip.open(file_path, 'rb'))) as f:
for text_line in f:
self.logfile_text_dict[self.index] = text_line.strip('\n')
diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py
index ea5ab23..b4e06b7 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -59,7 +59,9 @@ class CheckP(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.package = yield catpkgsplit(self.getProperty("cpv"))[1]
print(self.package)
- self.package_data = yield self.gentooci.db.packages.getPackageByName(self.package)
+ self.package_data = yield self.gentooci.db.packages.getPackageByName(self.package,
+ self.getProperty("category_data")['uuid'],
+ self.getProperty("repository_data")['uuid'])
print(self.package_data)
if self.package_data is None:
self.setProperty("package", self.package, 'package')
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py
index 426ddd4..e6a927a 100644
--- a/buildbot_gentoo_ci/steps/version.py
+++ b/buildbot_gentoo_ci/steps/version.py
@@ -38,7 +38,7 @@ class GetVData(BuildStep):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
self.version = yield cpv_getversion(self.getProperty("cpv"))
print(self.version)
- self.old_version_data = yield self.gentooci.db.versions.getVersionByName(self.version)
+ self.old_version_data = yield self.gentooci.db.versions.getVersionByName(self.version, self.getProperty("package_data")['uuid'])
print(self.old_version_data)
self.setProperty("old_version_data", self.old_version_data, 'old_version_data')
self.setProperty("version", self.version, 'version')
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
@ 2021-04-13 14:24 Magnus Granberg
0 siblings, 0 replies; 5+ messages in thread
From: Magnus Granberg @ 2021-04-13 14:24 UTC (permalink / raw
To: gentoo-commits
commit: 409167eb42b9fbacf05f00a36621e897361138bd
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Apr 13 14:24:15 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Apr 13 14:24:15 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=409167eb
Remove project_repository_uuid from db. use int for datetime in db. change mirror_url to url
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/builds.py | 7 +++----
buildbot_gentoo_ci/db/model.py | 32 ++++++++++++--------------------
buildbot_gentoo_ci/db/projects.py | 1 -
buildbot_gentoo_ci/db/repositorys.py | 2 +-
buildbot_gentoo_ci/db/versions.py | 6 ++----
buildbot_gentoo_ci/steps/builders.py | 2 +-
buildbot_gentoo_ci/steps/portage.py | 4 ++--
buildbot_gentoo_ci/steps/repos.py | 2 +-
8 files changed, 22 insertions(+), 34 deletions(-)
diff --git a/buildbot_gentoo_ci/db/builds.py b/buildbot_gentoo_ci/db/builds.py
index 79814b7..2ad67b8 100644
--- a/buildbot_gentoo_ci/db/builds.py
+++ b/buildbot_gentoo_ci/db/builds.py
@@ -23,13 +23,12 @@ import sqlalchemy as sa
from twisted.internet import defer
from buildbot.db import base
-from buildbot.util import epoch2datetime
class BuildsConnectorComponent(base.DBConnectorComponent):
#@defer.inlineCallbacks
def addBuild(self, project_build_data):
- created_at = epoch2datetime(int(self.master.reactor.seconds()))
+ created_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
tbl = self.db.model.projects_builds
# get the highest current number
@@ -56,7 +55,7 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
def setSatusBuilds(self, build_id, project_uuid, status):
- updated_at = epoch2datetime(int(self.master.reactor.seconds()))
+ updated_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
tbl = self.db.model.projects_builds
@@ -69,7 +68,7 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
def setBuildbotBuildIdBuilds(self, build_id, project_uuid, buildbot_build_id):
- updated_at = epoch2datetime(int(self.master.reactor.seconds()))
+ updated_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
tbl = self.db.model.projects_builds
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 54c964f..ede5cb7 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -70,7 +70,7 @@ class Model(base.DBConnectorComponent):
sa.Column('name', sa.String(255), nullable=False),
# description of the repository
sa.Column('description', sa.Text, nullable=True),
- sa.Column('mirror_url', sa.String(255), nullable=True),
+ sa.Column('url', sa.String(255), nullable=True),
sa.Column('type', sa.Enum('gitpuller'), nullable=False, default='gitpuller'),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
@@ -98,15 +98,11 @@ class Model(base.DBConnectorComponent):
"projects", metadata,
# unique id per project
sa.Column('uuid', sa.String(36), primary_key=True,
- default=lambda: str(uuid.uuid4()),
- ),
+ default=lambda: str(uuid.uuid4())),
# project's name
sa.Column('name', sa.String(255), nullable=False),
# description of the project
sa.Column('description', sa.Text, nullable=True),
- sa.Column('project_repository_uuid', sa.String(36),
- sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
- nullable=False),
sa.Column('profile', sa.String(255), nullable=False),
sa.Column('profile_repository_uuid', sa.String(36),
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
@@ -216,10 +212,10 @@ class Model(base.DBConnectorComponent):
sa.Column('buildbot_build_id', sa.Integer),
sa.Column('status', sa.Enum('failed','completed','in-progress','waiting', 'warning'), nullable=False),
sa.Column('requested', sa.Boolean, default=False),
- sa.Column('created_at', sa.DateTime, nullable=True),
- sa.Column('updated_at', sa.DateTime, nullable=True),
+ sa.Column('created_at', sa.Integer, nullable=True),
+ sa.Column('updated_at', sa.Integer, nullable=True),
sa.Column('deleted', sa.Boolean, default=False),
- sa.Column('deleted_at', sa.DateTime, nullable=True),
+ sa.Column('deleted_at', sa.Integer, nullable=True),
)
projects_pattern = sautils.Table(
@@ -247,16 +243,14 @@ class Model(base.DBConnectorComponent):
categorys = sautils.Table(
"categorys", metadata,
sa.Column('uuid', sa.String(36), primary_key=True,
- default=lambda: str(uuid.uuid4())
- ),
+ default=lambda: str(uuid.uuid4())),
sa.Column('name', sa.String(255), nullable=False),
)
packages = sautils.Table(
"packages", metadata,
sa.Column('uuid', sa.String(36), primary_key=True,
- default=lambda: str(uuid.uuid4()),
- ),
+ default=lambda: str(uuid.uuid4())),
sa.Column('name', sa.String(255), nullable=False),
sa.Column('category_uuid', sa.String(36),
sa.ForeignKey('categorys.uuid', ondelete='CASCADE'),
@@ -265,14 +259,13 @@ class Model(base.DBConnectorComponent):
sa.ForeignKey('repositorys.uuid', ondelete='CASCADE'),
nullable=False),
sa.Column('deleted', sa.Boolean, default=False),
- sa.Column('deleted_at', sa.DateTime, nullable=True),
+ sa.Column('deleted_at', sa.Integer, nullable=True),
)
versions = sautils.Table(
"versions", metadata,
sa.Column('uuid', sa.String(36), primary_key=True,
- default=lambda: str(uuid.uuid4()),
- ),
+ default=lambda: str(uuid.uuid4())),
sa.Column('name', sa.String(255), nullable=False),
sa.Column('package_uuid', sa.String(36),
sa.ForeignKey('packages.uuid', ondelete='CASCADE'),
@@ -280,15 +273,14 @@ class Model(base.DBConnectorComponent):
sa.Column('file_hash', sa.String(255), nullable=False),
sa.Column('commit_id', sa.String(255), nullable=False),
sa.Column('deleted', sa.Boolean, default=False),
- sa.Column('deleted_at', sa.DateTime, nullable=True),
+ sa.Column('deleted_at', sa.Integer, nullable=True),
)
versions_keywords = sautils.Table(
"versions_keywords", metadata,
# unique id per project
sa.Column('uuid', sa.String(36), primary_key=True,
- default=lambda: str(uuid.uuid4()),
- ),
+ default=lambda: str(uuid.uuid4())),
# project's name
sa.Column('keyword_id', sa.Integer,
sa.ForeignKey('keywords.id', ondelete='CASCADE')),
@@ -308,7 +300,7 @@ class Model(base.DBConnectorComponent):
sa.Column("uid", sa.Integer, primary_key=True),
# identifier (nickname) for this user; used for display
- sa.Column("identifier", sa.String(255), nullable=False),
+ sa.Column("email", sa.String(255), nullable=False),
# username portion of user credentials for authentication
sa.Column("bb_username", sa.String(128)),
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index 2edc14a..2393011 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -194,7 +194,6 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
uuid=row.uuid,
name=row.name,
description=row.description,
- project_repository_uuid=row.project_repository_uuid,
profile=row.profile,
profile_repository_uuid=row.profile_repository_uuid,
keyword_id=row.keyword_id,
diff --git a/buildbot_gentoo_ci/db/repositorys.py b/buildbot_gentoo_ci/db/repositorys.py
index 8806dd1..8bac1a1 100644
--- a/buildbot_gentoo_ci/db/repositorys.py
+++ b/buildbot_gentoo_ci/db/repositorys.py
@@ -95,7 +95,7 @@ class RepositorysConnectorComponent(base.DBConnectorComponent):
uuid=row.uuid,
name=row.name,
description=row.description,
- mirror_url=row.mirror_url,
+ url=row.url,
auto=row.auto,
enabled=row.enabled,
ebuild=row.ebuild,
diff --git a/buildbot_gentoo_ci/db/versions.py b/buildbot_gentoo_ci/db/versions.py
index 768c1dc..fcc19ee 100644
--- a/buildbot_gentoo_ci/db/versions.py
+++ b/buildbot_gentoo_ci/db/versions.py
@@ -23,8 +23,6 @@ import sqlalchemy as sa
from twisted.internet import defer
from buildbot.db import base
-from buildbot.util import epoch2datetime
-from buildbot.util import datetime2epoch
class VersionsConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
@@ -76,7 +74,7 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
@defer.inlineCallbacks
def delVersion(self, uuid):
- deleted_at = epoch2datetime(int(self.master.reactor.seconds()))
+ deleted_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
tbl = self.db.model.versions
@@ -111,5 +109,5 @@ class VersionsConnectorComponent(base.DBConnectorComponent):
file_hash=row.file_hash,
commit_id=row.commit_id,
deleted=row.deleted,
- deleted_at=datetime2epoch(row.deleted_at)
+ deleted_at=row.deleted_at
)
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index e51576f..d9817d8 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -300,7 +300,7 @@ class UpdateRepos(BuildStep):
repository_data = yield self.gentooci.db.repositorys.getRepositoryByUuid(project_repository_data['repository_uuid'])
repository_path = yield os.path.join(portage_repos_path, repository_data['name'])
yield self.build.addStepsAfterCurrentStep([
- steps.Git(repourl=repository_data['mirror_url'],
+ steps.Git(repourl=repository_data['url'],
mode='incremental',
submodules=True,
workdir=os.path.join(repository_path, ''))
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 2308617..c1a4d90 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -153,7 +153,7 @@ class SetReposConf(BuildStep):
repository_conf = []
repository_conf.append('[' + repository_data['name'] + ']')
repository_conf.append('location = ' + repository_path)
- repository_conf.append('sync-uri = ' + repository_data['mirror_url'])
+ repository_conf.append('sync-uri = ' + repository_data['url'])
repository_conf.append('sync-type = git')
repository_conf.append('auto-sync = no')
repository_conf_string = separator.join(repository_conf)
@@ -446,7 +446,7 @@ class SetReposConfLocal(BuildStep):
repository_conf = []
repository_conf.append('[' + self.getProperty("repository_data")['name'] + ']')
repository_conf.append('location = ' + repository_path)
- repository_conf.append('sync-uri = ' + self.getProperty("repository_data")['mirror_url'])
+ repository_conf.append('sync-uri = ' + self.getProperty("repository_data")['url'])
repository_conf.append('sync-type = git')
repository_conf.append('auto-sync = no')
yield WriteTextToFile(repos_conf_repository_path, repository_conf)
diff --git a/buildbot_gentoo_ci/steps/repos.py b/buildbot_gentoo_ci/steps/repos.py
index bc06e45..5b6b621 100644
--- a/buildbot_gentoo_ci/steps/repos.py
+++ b/buildbot_gentoo_ci/steps/repos.py
@@ -108,7 +108,7 @@ class CheckRepository(BuildStep):
repo_path = yield pygit2.discover_repository(repository_path)
print(repo_path)
if repo_path is None:
- yield pygit2.clone_repository(repository_data['mirror_url'], repository_path)
+ yield pygit2.clone_repository(repository_data['url'], repository_path)
success = True
else:
repo = yield pygit2.Repository(repo_path)
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
@ 2021-07-09 15:49 Magnus Granberg
0 siblings, 0 replies; 5+ messages in thread
From: Magnus Granberg @ 2021-07-09 15:49 UTC (permalink / raw
To: gentoo-commits
commit: c479075032be16a23ebbecf06d5f17bd8881e938
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Fri Jul 9 15:49:39 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Fri Jul 9 15:49:39 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c4790750
Move setting of project_build_data
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/builds.py | 1 -
buildbot_gentoo_ci/steps/builders.py | 30 ++++++++++++++++--------------
2 files changed, 16 insertions(+), 15 deletions(-)
diff --git a/buildbot_gentoo_ci/db/builds.py b/buildbot_gentoo_ci/db/builds.py
index 2ad67b8..2cd52bc 100644
--- a/buildbot_gentoo_ci/db/builds.py
+++ b/buildbot_gentoo_ci/db/builds.py
@@ -40,7 +40,6 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
q = tbl.insert()
r = conn.execute(q, dict(project_uuid=project_build_data['project_uuid'],
version_uuid=project_build_data['version_uuid'],
- buildbot_build_id=project_build_data['buildbot_build_id'],
status=project_build_data['status'],
requested=project_build_data['requested'],
created_at=created_at,
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 0bed647..63a1ae9 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -189,6 +189,17 @@ class TriggerRunBuildRequest(BuildStep):
@defer.inlineCallbacks
def run(self):
+ self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+ if self.getProperty('project_build_data') is None:
+ project_build_data = {}
+ project_build_data['project_uuid'] = self.getProperty('project_data')['uuid']
+ project_build_data['version_uuid'] = self.getProperty("version_data")['uuid']
+ project_build_data['status'] = 'waiting'
+ project_build_data['requested'] = False
+ project_build_data['id'], project_build_data['build_id'] = yield self.gentooci.db.builds.addBuild(
+ project_build_data)
+ else:
+ project_build_data = self.getProperty('project_build_data')
yield self.build.addStepsAfterCurrentStep([
steps.Trigger(
schedulerNames=['run_build_request'],
@@ -200,7 +211,7 @@ class TriggerRunBuildRequest(BuildStep):
'projectrepository_data' : self.getProperty('projectrepository_data'),
'use_data' : self.getProperty("use_data"),
'fullcheck' : self.getProperty("fullcheck"),
- 'project_build_data' : None
+ 'project_build_data' : project_build_data
}
)])
return SUCCESS
@@ -233,6 +244,7 @@ class GetProjectRepositoryData(BuildStep):
# set Property projectrepository_data so we can use it in the trigger
self.setProperty('projectrepository_data', projectrepository_data, 'projectrepository_data')
self.setProperty('use_data', None, 'use_data')
+ self.setProperty('project_data', project_data, 'project_data')
# get name o project keyword
project_keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
# if not * (all keywords)
@@ -280,22 +292,12 @@ class SetupPropertys(BuildStep):
self.setProperty('faild_version_data', None, 'faild_version_data')
self.setProperty('rerun', 0, 'rerun')
print(self.getProperty("buildnumber"))
- if self.getProperty('project_build_data') is None:
- project_build_data = {}
- project_build_data['project_uuid'] = project_data['uuid']
- project_build_data['version_uuid'] = self.getProperty("version_data")['uuid']
- project_build_data['status'] = 'in-progress'
- project_build_data['requested'] = False
- project_build_data['buildbot_build_id'] = self.getProperty("buildnumber")
- project_build_data['id'], project_build_data['build_id'] = yield self.gentooci.db.builds.addBuild(
- project_build_data)
- else:
- project_build_data = self.getProperty('project_build_data')
- yield self.gentooci.db.builds.setSatusBuilds(
+ project_build_data = self.getProperty('project_build_data')
+ yield self.gentooci.db.builds.setSatusBuilds(
project_build_data['build_id'],
project_build_data['project_uuid'],
'in-progress')
- yield self.gentooci.db.builds.setBuildbotBuildIdBuilds(
+ yield self.gentooci.db.builds.setBuildbotBuildIdBuilds(
project_build_data['build_id'],
project_build_data['project_uuid'],
self.getProperty("buildnumber"))
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
@ 2021-09-20 23:28 Magnus Granberg
0 siblings, 0 replies; 5+ messages in thread
From: Magnus Granberg @ 2021-09-20 23:28 UTC (permalink / raw
To: gentoo-commits
commit: a54c33a5119cb5528bb83ab73b86068dd749b970
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Sep 20 23:29:24 2021 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Sep 20 23:29:24 2021 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=a54c33a5
Support fore more make.conf options from the db
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/model.py | 1 +
buildbot_gentoo_ci/db/projects.py | 1 +
buildbot_gentoo_ci/steps/portage.py | 59 +++++++++----------------------------
3 files changed, 16 insertions(+), 45 deletions(-)
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index d9a3972..e2cbc8d 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -113,6 +113,7 @@ class Model(base.DBConnectorComponent):
sa.Column('status', sa.Enum('stable','unstable','all'), nullable=False),
sa.Column('auto', sa.Boolean, default=False),
sa.Column('enabled', sa.Boolean, default=False),
+ sa.Column('use_default', sa.Boolean, default=True),
sa.Column('created_by', sa.Integer,
sa.ForeignKey('users.uid', ondelete='CASCADE'),
nullable=False),
diff --git a/buildbot_gentoo_ci/db/projects.py b/buildbot_gentoo_ci/db/projects.py
index fbef435..408450e 100644
--- a/buildbot_gentoo_ci/db/projects.py
+++ b/buildbot_gentoo_ci/db/projects.py
@@ -211,6 +211,7 @@ class ProjectsConnectorComponent(base.DBConnectorComponent):
status=row.status,
auto=row.auto,
enabled=row.enabled,
+ use_default=row.use_default,
created_by=row.created_by
)
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py
index 0a131d7..03552b6 100644
--- a/buildbot_gentoo_ci/steps/portage.py
+++ b/buildbot_gentoo_ci/steps/portage.py
@@ -179,7 +179,7 @@ class SetMakeConf(BuildStep):
@defer.inlineCallbacks
def run(self):
- #FIXME: Make a dict before we pass it to the make.conf
+ #FIXME: Make a dict before we pass it to the log
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
project_data = self.getProperty('project_data')
makeconf_variables_data = yield self.gentooci.db.portages.getVariables()
@@ -189,46 +189,25 @@ class SetMakeConf(BuildStep):
for k in makeconf_variables_data:
makeconf_variables_values_data = yield self.gentooci.db.projects.getProjectMakeConfById(project_data['uuid'], k['id'])
makeconf_variable_list = []
- # we add some default values
- #FIXME:
- # we could set them in a config variables
- # FEATURES
- if k['variable'] == 'FEATURES':
- makeconf_variable_list.append('xattr')
- makeconf_variable_list.append('cgroup')
- makeconf_variable_list.append('-news')
- makeconf_variable_list.append('-collision-protect')
- makeconf_variable_list.append('split-log')
- makeconf_variable_list.append('compress-build-logs')
- # EMERGE_DEFAULT_OPTS
- if k['variable'] == 'EMERGE_DEFAULT_OPTS':
- makeconf_variable_list.append('--buildpkg=y')
- makeconf_variable_list.append('--rebuild-if-new-rev=y')
- makeconf_variable_list.append('--rebuilt-binaries=y')
- makeconf_variable_list.append('--usepkg=y')
- makeconf_variable_list.append('--binpkg-respect-use=y')
- makeconf_variable_list.append('--binpkg-changed-deps=y')
- makeconf_variable_list.append('--nospinner')
- makeconf_variable_list.append('--color=n')
- makeconf_variable_list.append('--ask=n')
- makeconf_variable_list.append('--quiet-build=y')
- makeconf_variable_list.append('--quiet-fail=y')
# CFLAGS
if k['variable'] == 'CFLAGS' or k['variable'] == 'FCFLAGS':
makeconf_variable_list.append('-O2')
makeconf_variable_list.append('-pipe')
- makeconf_variable_list.append('-march=native')
makeconf_variable_list.append('-fno-diagnostics-color')
#FIXME:
# Depend on worker we may have to add a diffrent march
+ makeconf_variable_list.append('-march=native')
if k['variable'] == 'CXXFLAGS':
makeconf_variable_list.append('${CFLAGS}')
if k['variable'] == 'FFLAGS':
makeconf_variable_list.append('${FCFLAGS}')
- if k['variable'] == 'ACCEPT_PROPERTIES':
- makeconf_variable_list.append('-interactive')
- if k['variable'] == 'ACCEPT_RESTRICT':
- makeconf_variable_list.append('-fetch')
+ # Add default setting if use_default
+ if project_data['use_default']:
+ default_project_data = yield self.gentooci.db.projects.getProjectByName(self.gentooci.config.project['project']['update_db'])
+ default_makeconf_variables_values_data = yield self.gentooci.db.projects.getProjectMakeConfById(default_project_data['uuid'], k['id'])
+ for v in default_makeconf_variables_values_data:
+ if v['build_id'] == 0:
+ makeconf_variable_list.append(v['value'])
for v in makeconf_variables_values_data:
if v['build_id'] == 0:
makeconf_variable_list.append(v['value'])
@@ -237,21 +216,10 @@ class SetMakeConf(BuildStep):
if makeconf_variable_list != []:
makeconf_variable_string = k['variable'] + '="' + separator2.join(makeconf_variable_list) + '"'
makeconf_list.append(makeconf_variable_string)
- # add hardcoded variables and values
- #FIXME:
- # we could set them in a config variables
- makeconf_list.append('LC_MESSAGES=C')
- makeconf_list.append('NOCOLOR="true"')
- makeconf_list.append('GCC_COLORS=""')
- makeconf_list.append('PORTAGE_TMPFS="/dev/shm"')
- makeconf_list.append('CLEAN_DELAY=0')
- makeconf_list.append('NOCOLOR=true')
- makeconf_list.append('PORT_LOGDIR="/var/cache/portage/logs"')
- makeconf_list.append('PKGDIR="/var/cache/portage/packages"')
- makeconf_list.append('DISTDIR="/var/cache/portage/distfiles"')
- makeconf_list.append('PORTAGE_ELOG_CLASSES="*"')
- # We need echo:info to get the logfile name
- makeconf_list.append('PORTAGE_ELOG_SYSTEM="save:* echo:info"')
+ # add hardcoded variables from config file
+ config_makeconfig = self.gentooci.config.project['project']['config_makeconfig']
+ for v in config_makeconfig:
+ makeconf_list.append(v)
# add ACCEPT_KEYWORDS from the project_data info
keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id'])
if project_data['status'] == 'unstable':
@@ -259,6 +227,7 @@ class SetMakeConf(BuildStep):
else:
makeconf_keyword = keyword_data['name']
makeconf_list.append('ACCEPT_KEYWORDS="' + makeconf_keyword + '"')
+ makeconf_list.append('MAKEOPTS="-j14"')
makeconf_string = separator1.join(makeconf_list)
print(makeconf_string)
yield self.build.addStepsAfterCurrentStep([
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/
@ 2022-09-25 11:46 Magnus Granberg
0 siblings, 0 replies; 5+ messages in thread
From: Magnus Granberg @ 2022-09-25 11:46 UTC (permalink / raw
To: gentoo-commits
commit: 3e54616897711b1958da178ca0578da08dd55cb3
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Sep 25 11:46:08 2022 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Sep 25 11:46:08 2022 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3e546168
Fix a typo on setStausBuilds
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org>
buildbot_gentoo_ci/db/builds.py | 2 +-
buildbot_gentoo_ci/steps/builders.py | 2 +-
buildbot_gentoo_ci/steps/logs.py | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/buildbot_gentoo_ci/db/builds.py b/buildbot_gentoo_ci/db/builds.py
index 6639ed5..57302ef 100644
--- a/buildbot_gentoo_ci/db/builds.py
+++ b/buildbot_gentoo_ci/db/builds.py
@@ -54,7 +54,7 @@ class BuildsConnectorComponent(base.DBConnectorComponent):
return self.db.pool.do(thd)
@defer.inlineCallbacks
- def setSatusBuilds(self, id, status):
+ def setStausBuilds(self, id, status):
updated_at = int(self.master.reactor.seconds())
def thd(conn, no_recurse=False):
diff --git a/buildbot_gentoo_ci/steps/builders.py b/buildbot_gentoo_ci/steps/builders.py
index 00980e3..aed9edb 100644
--- a/buildbot_gentoo_ci/steps/builders.py
+++ b/buildbot_gentoo_ci/steps/builders.py
@@ -333,7 +333,7 @@ class SetupPropertys(BuildStep):
project_build_data = self.getProperty('project_build_data')
project_build_data['status'] = 'in-progress'
project_build_data['buildbot_build_id'] = self.getProperty("buildnumber")
- yield self.gentooci.db.builds.setSatusBuilds(
+ yield self.gentooci.db.builds.setStausBuilds(
project_build_data['id'],
project_build_data['status'])
yield self.gentooci.db.builds.setBuildbotBuildIdBuilds(
diff --git a/buildbot_gentoo_ci/steps/logs.py b/buildbot_gentoo_ci/steps/logs.py
index 57f9394..becf7d1 100644
--- a/buildbot_gentoo_ci/steps/logs.py
+++ b/buildbot_gentoo_ci/steps/logs.py
@@ -596,7 +596,7 @@ class setBuildStatus(BuildStep):
def run(self):
self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
project_build_data = self.getProperty('project_build_data')
- yield self.gentooci.db.builds.setSatusBuilds(
+ yield self.gentooci.db.builds.setStausBuilds(
project_build_data['id'],
self.getProperty('status')
)
^ permalink raw reply related [flat|nested] 5+ messages in thread
end of thread, other threads:[~2022-09-25 11:46 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2021-04-10 1:03 [gentoo-commits] proj/tinderbox-cluster:master commit in: buildbot_gentoo_ci/steps/, buildbot_gentoo_ci/db/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2021-04-13 14:24 Magnus Granberg
2021-07-09 15:49 Magnus Granberg
2021-09-20 23:28 Magnus Granberg
2022-09-25 11:46 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox