* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2015-08-05 1:10 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2015-08-05 1:10 UTC (permalink / raw
To: gentoo-commits
commit: 1cbba36a9ea54c96b751728c98e30e0c23fac006
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Aug 5 01:09:51 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Aug 5 01:09:51 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1cbba36a
add remove package and fix some repoman stuff
pym/tbc/build_log.py | 4 ++--
pym/tbc/package.py | 38 ++++++++++++++++++++++----------------
pym/tbc/sqlquerys.py | 5 +++++
3 files changed, 29 insertions(+), 18 deletions(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 71c14b5..d72ef91 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -32,7 +32,7 @@ from tbc.sqlquerys import add_logs, get_config_id, get_ebuild_id_db, add_new_bui
add_repoman_log
from sqlalchemy.orm import sessionmaker
-def repoman_check_full(session, pkgdir, package_id, config_id):
+def check_repoman_full(session, pkgdir, package_id, config_id):
# Check cp with repoman repoman full
status = repoman_full(session, pkgdir, config_id)
repoman_hash = hashlib.sha256()
@@ -224,7 +224,7 @@ def get_buildlog_info(session, settings, pkg, build_dict):
categories = element[0]
package = element[1]
pkgdir = myportdb.getRepositoryPath(build_dict['repo']) + "/" + categories + "/" + package
- repoman_error_list = repoman_check_full(session, pkgdir, build_dict['package_id'], config_id)
+ repoman_error_list = check_repoman_full(session, pkgdir, build_dict['package_id'], config_id)
build_log_dict = {}
build_log_dict['fail'] = False
if repoman_error_list:
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 657c90d..0bb41eb 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -16,7 +16,7 @@ from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
get_package_metadata_sql, update_package_metadata, update_manifest_sql, \
get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \
get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info, \
- get_ebuild_info_ebuild_id, get_ebuild_restrictions
+ get_ebuild_info_ebuild_id, get_ebuild_restrictions, add_old_package
class tbc_package(object):
@@ -159,6 +159,7 @@ class tbc_package(object):
log_msg = "B %s:%s USE: %s Setup: %s" % (k, v['repo'], use_flagsDict, setup_id,)
add_logs(self._session, log_msg, "info", self._config_id)
i = i +1
+
def get_changelog_text(self, pkgdir):
changelog_text_dict, max_text_lines = get_log_text_dict(pkgdir + "/ChangeLog")
spec = 3
@@ -214,7 +215,7 @@ class tbc_package(object):
def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree):
# Use packageDict to update the db
ebuild_id_list = add_new_ebuild_sql(self._session, packageDict)
-
+
# Make old ebuilds unactive
for ebuild_id in ebuild_id_list:
new_ebuild_id_list.append(ebuild_id)
@@ -286,12 +287,13 @@ class tbc_package(object):
return
package_id = add_new_package_sql(self._session, cp, repo)
-
+
# Check cp with repoman full
status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
if status:
- log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
- add_logs(self._session, log_msg, "error", self._config_id)
+ log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
+ add_logs(self._session, log_msg, "error", self._config_id)
+
package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
# Get the ebuild list for cp
ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
@@ -308,7 +310,7 @@ class tbc_package(object):
old_ebuild_id_list = []
for cpv in sorted(ebuild_list_tree):
packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
-
+
# take package descriptions from the ebuilds
if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
@@ -329,6 +331,7 @@ class tbc_package(object):
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
mytree = []
mytree.append(repodir)
+
manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
if manifest_checksum_tree is None:
return
@@ -340,17 +343,11 @@ class tbc_package(object):
log_msg = "U %s:%s" % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
- # Check cp with repoman full
- status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
- if status:
- log_msg = "Repoman %s:%s ... Fail." % (cpv, repo)
- add_logs(self._session, log_msg, "error", self._config_id)
-
# Get the ebuild list for cp
old_ebuild_id_list = []
ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
if ebuild_list_tree == []:
- if manifest_checksum_tree != "0":
+ if manifest_checksum_tree == "0":
old_ebuild_id_list = get_ebuild_id_list(self._session, package_id)
for ebuild_id in old_ebuild_id_list:
EbuildInfo = get_ebuild_info_ebuild_id(self._session, ebuild_id)
@@ -358,15 +355,24 @@ class tbc_package(object):
# R = remove ebuild
log_msg = "R %s:%s" % (cpv, repo,)
add_logs(self._session, log_msg, "info", self._config_id)
+ if not os.path.isdir(pkgdir):
+ add_old_package(self._session, package_id)
add_old_ebuild(self._session, old_ebuild_id_list)
log_msg = "C %s:%s ... Done." % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
+
else:
log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
add_logs(self._session, log_msg, "info", self._config_id)
log_msg = "C %s:%s ... Fail." % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
return
+
+ # Check cp with repoman full
+ status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
+ if status:
+ log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
+ add_logs(self._session, log_msg, "error", self._config_id)
package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
packageDict ={}
new_ebuild_id_list = []
@@ -374,14 +380,14 @@ class tbc_package(object):
# split out ebuild version
ebuild_version_tree = portage.versions.cpv_getversion(cpv)
-
+
# Get packageDict for cpv
packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
-
+
# take package descriptions from the ebuilds
if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
-
+
# Get the checksum of the ebuild in tree and db
ebuild_version_checksum_tree = packageDict[cpv]['checksum']
checksums_db, fail = get_ebuild_checksums(self._session, package_id, ebuild_version_tree)
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 4d2a5e5..7791e6e 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -453,6 +453,11 @@ def add_old_ebuild(session, old_ebuild_list):
for build_job in build_job_id_list:
del_old_build_jobs(session, build_job.BuildJobId)
+def add_old_package(session, package_id):
+ PackagesInfo = session.query(Packages).filter_by(PackageId = package_id).one()
+ PackagesInfo.Active = False
+ session.commit()
+
def add_new_package_sql(session, cp, repo):
element = cp.split('/')
categories = element[0]
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2015-09-03 21:03 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2015-09-03 21:03 UTC (permalink / raw
To: gentoo-commits
commit: aac3ed04e3bc5d555b2432ea40dd262335dfd776
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Sep 3 21:02:29 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Sep 3 21:02:29 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=aac3ed04
Add support move from cvs to git
pym/tbc/db_mapping.py | 7 ++--
pym/tbc/package.py | 91 ++++++++++++++++++-----------------------------
pym/tbc/qachecks.py | 97 ---------------------------------------------------
pym/tbc/sync.py | 17 ++++-----
4 files changed, 44 insertions(+), 168 deletions(-)
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index dbae5a8..82318a1 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -103,7 +103,7 @@ class PackagesEmails(Base):
class PackagesMetadata(Base):
Id = Column('id', Integer, primary_key=True)
PackageId = Column('package_id', Integer, ForeignKey('packages.package_id'))
- Changlog = Column('changlog', Text)
+ Gitlog = Column('gitlog', Text)
Descriptions = Column('descriptions', Text)
__tablename__ = 'packages_metadata'
@@ -278,6 +278,7 @@ class EbuildsKeywords(Base):
class EbuildsMetadata(Base):
Id = Column('id', Integer, primary_key=True)
EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
- Revision = Column('revision', String(30))
- Descriptions = Column('descriptions', String(200))
+ Commit = Column('commit', String(100))
+ New = Column('new', Boolean, default=False)
+ Descriptions = Column('descriptions', Text)
__tablename__ = 'ebuilds_metadata'
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 980b089..6224846 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -5,12 +5,13 @@ from __future__ import print_function
import re
import hashlib
import os
+import git
import portage
from portage.xml.metadata import MetaDataXML
from tbc.flags import tbc_use_flags
from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
from tbc.flags import tbc_use_flags
-from tbc.qachecks import digestcheck, check_repoman
+from tbc.qachecks import check_repoman
from tbc.build_log import check_repoman_full
from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
@@ -99,9 +100,10 @@ class tbc_package(object):
add_logs(self._session, log_msg, "info", self._config_id)
log_msg = "C %s:%s ... Fail." % (cpv, repo)
add_logs(self._session, log_msg, "info", self._config_id)
- ebuild_version_cvs_revision_tree = '0'
+ git_commit = '0'
else:
- ebuild_version_cvs_revision_tree = get_ebuild_cvs_revision(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")
+ git_commit = '0'
+ #FIXME The git commit need to be fixed or when $ID get fixed
# Get the ebuild metadata
ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv, repo)
@@ -123,7 +125,8 @@ class tbc_package(object):
attDict['checksum']= ebuild_version_checksum_tree
attDict['ebuild_version_metadata_tree'] = ebuild_version_metadata_tree
#attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
- attDict['ebuild_version_revision_tree'] = ebuild_version_cvs_revision_tree
+ attDict['git_commit'] = git_commit
+ attDict['New'] = False
attDict['ebuild_version_descriptions_tree'] = ebuild_version_metadata_tree[7]
return attDict
@@ -161,33 +164,12 @@ class tbc_package(object):
add_logs(self._session, log_msg, "info", self._config_id)
i = i +1
- def get_changelog_text(self, pkgdir):
- changelog_text_dict, max_text_lines = get_log_text_dict(pkgdir + "/ChangeLog")
- spec = 3
- spec_tmp = 1
- changelog_text_tree = ''
- for index, text_line in changelog_text_dict.items():
- if index == max_text_lines:
- if not re.search('^\n', text_line):
- changelog_text_tree = changelog_text_tree + text_line
- break
- elif re.search('^#', text_line):
- pass
- elif re.search('^\n', text_line) and re.search('^#', changelog_text_dict[index - 1]):
- pass
- elif re.search('^\n', text_line) and re.search('^\*', changelog_text_dict[index + 1]):
- changelog_text_tree = changelog_text_tree + text_line
- spec_tmp = spec_tmp + 1
- spec = spec + 1
- elif re.search('^\n', text_line) and not re.search('^\*', changelog_text_dict[index + 1]):
- if spec_tmp == spec:
- break
- else:
- spec_tmp = spec_tmp + 1
- changelog_text_tree = changelog_text_tree + text_line
- else:
- changelog_text_tree = changelog_text_tree + text_line
- return changelog_text_tree
+ def get_git_log_pkg_text(self, repodir, cp):
+ n = '5'
+ git_log_pkg = ''
+ g = git.Git(repodir)
+ git_log_pkg = g.log('-n ' + n, '--grep=' + cp)
+ return git_log_pkg
def get_package_metadataDict(self, pkgdir, package_id):
# Make package_metadataDict
@@ -195,20 +177,26 @@ class tbc_package(object):
package_metadataDict = {}
md_email_list = []
herd = None
- pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
- attDict['changelog_text'] = self.get_changelog_text(pkgdir)
- tmp_herds = pkg_md.herds()
- if tmp_herds != ():
- attDict['metadata_xml_herds'] = tmp_herds[0]
- md_email_list.append(attDict['metadata_xml_herds'] + '@gentoo.org')
- for maint in pkg_md.maintainers():
- md_email_list.append(maint.email)
- if md_email_list != []:
- attDict['metadata_xml_email'] = md_email_list
- else:
- log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
+ try:
+ pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
+ except:
+ log_msg = "Metadata file %s is missing or has errors" % (pkgdir + "/metadata.xml")
add_logs(self._session, log_msg, "qa", self._config_id)
attDict['metadata_xml_email'] = False
+ else:
+ attDict['git_log_pkg_text'] = self.get_git_log_pkg_text(repodir, cp)
+ tmp_herds = pkg_md.herds()
+ if tmp_herds != ():
+ attDict['metadata_xml_herds'] = tmp_herds[0]
+ md_email_list.append(attDict['metadata_xml_herds'] + '@gentoo.org')
+ for maint in pkg_md.maintainers():
+ md_email_list.append(maint.email)
+ if md_email_list != []:
+ attDict['metadata_xml_email'] = md_email_list
+ else:
+ log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
+ add_logs(self._session, log_msg, "qa", self._config_id)
+ attDict['metadata_xml_email'] = False
attDict['metadata_xml_descriptions'] = ''
package_metadataDict[package_id] = attDict
return package_metadataDict
@@ -248,7 +236,6 @@ class tbc_package(object):
self.add_new_build_job_db(ebuild_id_list, packageDict, config_cpv_listDict)
def get_manifest_checksum_tree(self, pkgdir, cp, repo, mytree):
-
# Get the cp manifest file checksum.
try:
manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0]
@@ -261,13 +248,6 @@ class tbc_package(object):
log_msg = "C %s:%s ... Fail." % (cp, repo)
add_logs(self._session, log_msg, "error", self._config_id)
return "0"
- fail_msg = digestcheck(self._mysettings, pkgdir)
- if fail_msg:
- log_msg = "QA: Manifest file has errors. :%s:%s" % (cp, repo,)
- add_logs(self._session, log_msg, "error", self._config_id)
- log_msg = "C %s:%s ... Fail." % (cp, repo)
- add_logs(self._session, log_msg, "error", self._config_id)
- return None
return manifest_checksum_tree
def add_new_package_db(self, cp, repo):
@@ -282,10 +262,7 @@ class tbc_package(object):
mytree = []
mytree.append(repodir)
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
-
manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
- if manifest_checksum_tree is None:
- return
package_id = add_new_package_sql(self._session, cp, repo)
@@ -316,6 +293,7 @@ class tbc_package(object):
if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
+ packageDict[cpv]['new'] = True
self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
@@ -332,10 +310,7 @@ class tbc_package(object):
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
mytree = []
mytree.append(repodir)
-
manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
- if manifest_checksum_tree is None:
- return
# if we NOT have the same checksum in the db update the package
if manifest_checksum_tree != PackageInfo.Checksum:
@@ -361,7 +336,6 @@ class tbc_package(object):
add_old_ebuild(self._session, old_ebuild_id_list)
log_msg = "C %s:%s ... Done." % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
-
else:
log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
add_logs(self._session, log_msg, "info", self._config_id)
@@ -414,6 +388,7 @@ class tbc_package(object):
# N = New ebuild
log_msg = "N %s:%s" % (cpv, repo,)
add_logs(self._session, log_msg, "info", self._config_id)
+ packageDict[cpv]['new'] = True
elif ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
# U = Updated ebuild
log_msg = "U %s:%s" % (cpv, repo,)
diff --git a/pym/tbc/qachecks.py b/pym/tbc/qachecks.py
index 766b572..67b1448 100644
--- a/pym/tbc/qachecks.py
+++ b/pym/tbc/qachecks.py
@@ -19,103 +19,6 @@ from tbc.repoman.main import repoman_main
from tbc.sqlquerys import get_configmetadata_info, get_config_info, get_setup_info
import portage
-# Copy of portage.digestcheck() but without the writemsg() stuff
-def digestcheck(mysettings, pkgdir):
- """
- Verifies checksums. Assumes all files have been downloaded.
- @rtype: int
- @return: 1 on success and 0 on failure
- """
- strict=False
- myfiles = []
- justmanifest = None
- mf = None
-
- if mysettings.get("EBUILD_SKIP_MANIFEST") == "1":
- return False
- hash_filter = _hash_filter(mysettings.get("PORTAGE_CHECKSUM_FILTER", ""))
- if hash_filter.transparent:
- hash_filter = None
- if mf is None:
- mf = mysettings.repositories.get_repo_for_location(
- os.path.dirname(os.path.dirname(pkgdir)))
- mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"])
- try:
- if not mf.thin and strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings:
- if mf.fhashdict.get("EBUILD"):
- mf.checkTypeHashes("EBUILD", hash_filter=hash_filter)
- if mf.fhashdict.get("AUX"):
- mf.checkTypeHashes("AUX", hash_filter=hash_filter)
- if mf.fhashdict.get("MISC"):
- mf.checkTypeHashes("MISC", ignoreMissingFiles=True,
- hash_filter=hash_filter)
- for f in myfiles:
- ftype = mf.findFile(f)
- if ftype is None:
- if mf.allow_missing:
- continue
- return ("\n!!! Missing digest for '%s'\n") % f
- mf.checkFileHashes(ftype, f, hash_filter=hash_filter)
- except FileNotFound as e:
- return ("\n!!! A file listed in the Manifest could not be found: %s\n") % str(e)
- except DigestException as e:
- return ("!!! Digest verification failed: %s\nReason: %s\nGot: %s\nExpected: %s\n") \
- % (e.value[0], e.value[1], e.value[2], e.value[3])
- if mf.thin or mf.allow_missing:
- # In this case we ignore any missing digests that
- # would otherwise be detected below.
- return False
- # Make sure that all of the ebuilds are actually listed in the Manifest.
- for f in os.listdir(pkgdir):
- pf = None
- if f[-7:] == '.ebuild':
- pf = f[:-7]
- if pf is not None and not mf.hasFile("EBUILD", f):
- return ("!!! A file is not listed in the Manifest: '%s'\n") % os.path.join(pkgdir, f)
-
- # epatch will just grab all the patches out of a directory, so we have to
- # make sure there aren't any foreign files that it might grab.
- filesdir = os.path.join(pkgdir, "files")
-
- for parent, dirs, files in os.walk(filesdir):
- try:
- parent = _unicode_decode(parent,
- encoding=_encodings['fs'], errors='strict')
- except UnicodeDecodeError:
- parent = _unicode_decode(parent,
- encoding=_encodings['fs'], errors='replace')
- return ("!!! Path contains invalid character(s) for encoding '%s': '%s'\n") % (_encodings['fs'], parent)
- for d in dirs:
- d_bytes = d
- try:
- d = _unicode_decode(d,
- encoding=_encodings['fs'], errors='strict')
- except UnicodeDecodeError:
- d = _unicode_decode(d,
- encoding=_encodings['fs'], errors='replace')
- return ("!!! Path contains invalid character(s) for encoding '%s': '%s'\n") % (_encodings['fs'], os.path.join(parent, d))
- if d.startswith(".") or d == "CVS":
- dirs.remove(d_bytes)
- for f in files:
- try:
- f = _unicode_decode(f,
- encoding=_encodings['fs'], errors='strict')
- except UnicodeDecodeError:
- f = _unicode_decode(f,
- encoding=_encodings['fs'], errors='replace')
- if f.startswith("."):
- continue
- f = os.path.join(parent, f)[len(filesdir) + 1:]
- return ("!!! File name contains invalid character(s) for encoding '%s': '%s'\n") % (_encodings['fs'], f)
- if f.startswith("."):
- continue
- f = os.path.join(parent, f)[len(filesdir) + 1:]
- file_type = mf.findFile(f)
- if file_type != "AUX" and not f.startswith("digest-"):
- return ("!!! A file is not listed in the Manifest: '%s'\n") % os.path.join(filesdir, f)
- return False
-
-
def check_file_in_manifest(pkgdir, mysettings, portdb, cpv, build_use_flags_list, repo):
myfetchlistdict = portage.FetchlistDict(pkgdir, mysettings, portdb)
my_manifest = portage.Manifest(pkgdir, mysettings['DISTDIR'], fetchlist_dict=myfetchlistdict, manifest1_compat=False, from_scratch=False)
diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 12c4c9f..610f357 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -74,20 +74,16 @@ def git_sync_main(session):
for repo_dir in git_repos_list(myportdb):
reponame = myportdb.getRepositoryName(repo_dir)
repo = git.Repo(repo_dir)
+ log_msg = "Checking repo %s" % (reponame)
+ add_logs(session, log_msg, "info", config_id)
info_list, repouptodate = git_fetch(repo)
if not repouptodate:
cp_list = []
attr = {}
# We check for Manifest changes and add the package to a list
- for diff_line in repo.git.diff('origin').splitlines():
- if re.search("^diff --git.*/Manifest", diff_line):
- diff_line2 = re.split(' b/', re.sub('diff --git', '', diff_line))
- diff_line3 = re.sub(' a/', '', diff_line2[0])
- if diff_line3 == diff_line2[1] or "Manifest" in diff_line3:
- cp = re.sub('/Manifest', '', diff_line3)
- cp_list.append(cp)
- else:
- cp = re.sub('/Manifest', '', diff_line2[1])
+ for diff_line in repo.git.diff('origin', '--name-only').splitlines():
+ if re.search("Manifest$", diff_line):
+ cp = re.sub('/Manifest', '', diff_line)
cp_list.append(cp)
attr['cp_list'] = cp_list
repo_cp_dict[reponame] = attr
@@ -95,7 +91,8 @@ def git_sync_main(session):
else:
log_msg = "Repo %s is up to date" % (reponame)
add_logs(session, log_msg, "info", config_id)
-
+ log_msg = "Checking repo %s Done" % (reponame)
+ add_logs(session, log_msg, "info", config_id)
# Need to add a clone of profiles/base for reading the tree
try:
os.mkdir(mysettings['PORTDIR'] + "/profiles/config", 0o777)
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2015-09-06 20:29 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2015-09-06 20:29 UTC (permalink / raw
To: gentoo-commits
commit: 988511359bd3f58aa91f8cd94179b8f44779719a
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Sep 6 20:28:38 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Sep 6 20:28:38 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=98851135
add git log file commit ID
pym/tbc/package.py | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 6224846..ef9b453 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -82,6 +82,13 @@ class tbc_package(object):
if ebuild_auxdb_list[i] == '':
ebuild_auxdb_list[i] = ''
return ebuild_auxdb_list
+ def get_git_log_ebuild(self, repodir, ebuild_file):
+ git_log_ebuild = ''
+ g = git.Git(repodir)
+ for line in g.log('-n 1', ebuild_file).splitlines():
+ if re.search('^commit', line):
+ git_log_ebuild = re.sub('commit', '', line)
+ return git_log_ebuild
def get_packageDict(self, pkgdir, cpv, repo):
@@ -90,10 +97,10 @@ class tbc_package(object):
element = portage.versions.cpv_getkey(cpv).split('/')
categories = element[0]
package = element[1]
-
+ ebuild_file = pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild"
# Make a checksum of the ebuild
try:
- ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild")[0]
+ ebuild_version_checksum_tree = portage.checksum.sha256hash(ebuild_file)[0]
except:
ebuild_version_checksum_tree = "0"
log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,)
@@ -102,8 +109,7 @@ class tbc_package(object):
add_logs(self._session, log_msg, "info", self._config_id)
git_commit = '0'
else:
- git_commit = '0'
- #FIXME The git commit need to be fixed or when $ID get fixed
+ git_commit = self.get_git_log_ebuild(repodir, ebuild_file):
# Get the ebuild metadata
ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv, repo)
@@ -177,14 +183,13 @@ class tbc_package(object):
package_metadataDict = {}
md_email_list = []
herd = None
+ attDict['metadata_xml_email'] = False
try:
pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
except:
log_msg = "Metadata file %s is missing or has errors" % (pkgdir + "/metadata.xml")
add_logs(self._session, log_msg, "qa", self._config_id)
- attDict['metadata_xml_email'] = False
else:
- attDict['git_log_pkg_text'] = self.get_git_log_pkg_text(repodir, cp)
tmp_herds = pkg_md.herds()
if tmp_herds != ():
attDict['metadata_xml_herds'] = tmp_herds[0]
@@ -196,7 +201,7 @@ class tbc_package(object):
else:
log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
add_logs(self._session, log_msg, "qa", self._config_id)
- attDict['metadata_xml_email'] = False
+ attDict['git_log_pkg_text'] = self.get_git_log_pkg_text(repodir, cp)
attDict['metadata_xml_descriptions'] = ''
package_metadataDict[package_id] = attDict
return package_metadataDict
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2015-10-06 19:31 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2015-10-06 19:31 UTC (permalink / raw
To: gentoo-commits
commit: 6698f841f9f77bad00a963ae61bf0625d5ce35c3
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 6 19:31:35 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Oct 6 19:31:35 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=6698f841
add more repoman fixes
pym/tbc/package.py | 2 +-
pym/tbc/qachecks.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index ef9b453..091f75f 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -109,7 +109,7 @@ class tbc_package(object):
add_logs(self._session, log_msg, "info", self._config_id)
git_commit = '0'
else:
- git_commit = self.get_git_log_ebuild(repodir, ebuild_file):
+ git_commit = self.get_git_log_ebuild(repodir, ebuild_file)
# Get the ebuild metadata
ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv, repo)
diff --git a/pym/tbc/qachecks.py b/pym/tbc/qachecks.py
index 67b1448..451f315 100644
--- a/pym/tbc/qachecks.py
+++ b/pym/tbc/qachecks.py
@@ -14,7 +14,7 @@ from portage import os, _encodings, _unicode_decode, _unicode_encode
from portage.exception import DigestException, FileNotFound, ParseError, PermissionDenied
from _emerge.Package import Package
from _emerge.RootConfig import RootConfig
-from tbc.repoman.checks.ebuilds.checks import run_checks
+from repoman.checks.ebuilds.checks import run_checks
from tbc.repoman.main import repoman_main
from tbc.sqlquerys import get_configmetadata_info, get_config_info, get_setup_info
import portage
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2015-12-20 0:12 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2015-12-20 0:12 UTC (permalink / raw
To: gentoo-commits
commit: 9d888380b877c7e310fc7799962776d17a242821
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Dec 20 00:12:08 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Dec 20 00:12:08 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=9d888380
fix for thin manifest
pym/tbc/qachecks.py | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/pym/tbc/qachecks.py b/pym/tbc/qachecks.py
index acb080e..892642f 100644
--- a/pym/tbc/qachecks.py
+++ b/pym/tbc/qachecks.py
@@ -22,10 +22,6 @@ import portage
def check_file_in_manifest(pkgdir, mysettings, portdb, cpv, build_use_flags_list, repo):
myfetchlistdict = portage.FetchlistDict(pkgdir, mysettings, portdb)
my_manifest = portage.Manifest(pkgdir, mysettings['DISTDIR'], fetchlist_dict=myfetchlistdict, manifest1_compat=False, from_scratch=False)
- ebuild_version = portage.versions.cpv_getversion(cpv)
- package = portage.versions.cpv_getkey(cpv).split("/")[1]
- if my_manifest.findFile(package + "-" + ebuild_version + ".ebuild") is None:
- return "Ebuild file not found.\n"
tree = portdb.getRepositoryPath(repo)
cpv_fetchmap = portdb.getFetchMap(cpv, useflags=build_use_flags_list, mytree=tree)
mysettings.unlock()
@@ -37,12 +33,12 @@ def check_file_in_manifest(pkgdir, mysettings, portdb, cpv, build_use_flags_list
finally:
mysettings.lock()
try:
- my_manifest.checkCpvHashes(cpv, checkDistfiles=True, onlyDistfiles=False, checkMiscfiles=True)
+ my_manifest.checkCpvHashes(cpv, checkDistfiles=True, onlyDistfiles=True, checkMiscfiles=False)
except:
return "Can't fetch the file or the hash failed.\n"
try:
portdb.fetch_check(cpv, useflags=build_use_flags_list, mysettings=mysettings, all=False)
- except:
+ except:
return "Fetch check failed.\n"
return
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2015-12-21 22:54 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2015-12-21 22:54 UTC (permalink / raw
To: gentoo-commits
commit: 8f4a556c399d1efd129a5a339c1d65e0801d1471
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Dec 21 22:53:31 2015 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Dec 21 22:53:31 2015 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8f4a556c
add support for irker
pym/tbc/build_log.py | 7 ++++++-
pym/tbc/irk.py | 28 ++++++++++++++++++++++++++++
2 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index bbe35d4..52db083 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -19,7 +19,7 @@ from _emerge.main import parse_opts
portage.proxy.lazyimport.lazyimport(globals(),
'tbc.actions:action_info,load_emerge_config',
)
-
+from tbc.irk import send_irk
from tbc.qachecks import check_repoman, repoman_full
from tbc.text import get_log_text_dict
from tbc.readconf import read_config_settings
@@ -301,6 +301,11 @@ def add_buildlog_main(settings, pkg, trees):
log_msg = "Package: %s:%s is logged." % (pkg.cpv, pkg.repo,)
add_logs(session, log_msg, "info", config_id)
print("\n>>> Logging %s:%s\n" % (pkg.cpv, pkg.repo,))
+ if build_log_dict['fail']:
+ msg = "Package: %s Repo: %s FAILD Weblink http://foo.gg.oo/buildpackage/%s\n" % (pkg.cpv, pkg.repo, log_id,)
+ else:
+ msg = "Package: %s Repo: %s PASS Weblink http://foo.gg.oo/buildpackage/%s\n" % (pkg.cpv, pkg.repo, log_id,)
+ send_irk(msg)
session.close
def log_fail_queru(session, build_dict, settings):
diff --git a/pym/tbc/irk.py b/pym/tbc/irk.py
new file mode 100644
index 0000000..01765c7
--- /dev/null
+++ b/pym/tbc/irk.py
@@ -0,0 +1,28 @@
+from __future__ import unicode_literals
+
+import json
+import socket
+import sys
+
+DEFAULT_SERVER = ("192.168.0.5", 6659)
+
+def connect(server = DEFAULT_SERVER):
+ return socket.create_connection(server)
+
+def send(s, target, message):
+ data = {"to": target, "privmsg" : message}
+ s.sendall(json.dumps(data).encode('ascii'))
+
+def irk(target, message, server = DEFAULT_SERVER):
+ s = connect(server)
+ if "irc:" not in target and "ircs:" not in target:
+ target = "irc://chat.freenode.net/{0}".format(target)
+ send(s, target, message)
+ s.close()
+
+def send_irk(msg):
+ target = "tinderbox-cluster"
+ try:
+ irk(target, msg)
+ except socket.error as e:
+ sys.stderr.write("irk: write to server failed: %r\n" % e)
\ No newline at end of file
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-01-07 7:33 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-01-07 7:33 UTC (permalink / raw
To: gentoo-commits
commit: 2e76eff16ecad10b5106080359e49c896bea950f
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 7 07:35:20 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Jan 7 07:35:20 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2e76eff1
add support for repoman cpv
pym/tbc/build_log.py | 52 +++++++++++++++++++++++++++++++++++-----------------
1 file changed, 35 insertions(+), 17 deletions(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 52db083..79dbe2a 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -32,25 +32,35 @@ from tbc.sqlquerys import add_logs, get_config_id, get_ebuild_id_db, add_new_bui
add_repoman_log
from sqlalchemy.orm import sessionmaker
-def check_repoman_full(session, pkgdir, package_id, config_id):
+def check_repoman_full(session, pkgdir, package_id, config_id, cpv=False):
# Check cp with repoman repoman full
status = repoman_full(session, pkgdir, config_id)
repoman_hash = hashlib.sha256()
- repoman_log = ""
+ if cpv:
+ element = cpv.split('/')
+ pv = element[1]
if status:
+ repoman_dict = {}
for k, v in status.items():
- repoman_line = k + '/n'
- repoman_hash.update(repoman_line.encode('utf-8'))
- repoman_log = repoman_log + repoman_line
+ repoman_log2 = []
for line in v:
- repoman_line = line + '/n'
- repoman_hash.update(repoman_line.encode('utf-8'))
- repoman_log = repoman_log + repoman_line
- add_repoman_log(session, package_id, repoman_log, repoman_hash.hexdigest())
- return repoman_log
- else:
- return status
-
+ if not cpv:
+ repoman_log2.append(line)
+ elif cpv and re.search(pv + '.ebuild', line):
+ repoman_log2.append(line)
+ if not repoman_log2 == []:
+ repoman_dict[k] = repoman_log2
+ if not repoman_dict == {}:
+ repoman_log = ""
+ for k, v in repoman_dict.items():
+ repoman_log = repoman_log + k + "\n"
+ repoman_hash.update(k.encode('utf-8'))
+ for line in v:
+ repoman_log = repoman_log + line + "\n"
+ repoman_hash.update(line.encode('utf-8'))
+ add_repoman_log(session, package_id, repoman_log, repoman_hash.hexdigest())
+ return repoman_log
+ return False
def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
myportdb = portage.portdbapi(mysettings=settings)
@@ -224,7 +234,7 @@ def get_buildlog_info(session, settings, pkg, build_dict, config_id):
categories = element[0]
package = element[1]
pkgdir = myportdb.getRepositoryPath(build_dict['repo']) + "/" + categories + "/" + package
- repoman_error_list = check_repoman_full(session, pkgdir, build_dict['package_id'], config_id)
+ repoman_error_list = check_repoman_full(session, pkgdir, build_dict['package_id'], config_id, build_dict['cpv'])
build_log_dict = {}
build_log_dict['fail'] = False
if repoman_error_list:
@@ -301,10 +311,18 @@ def add_buildlog_main(settings, pkg, trees):
log_msg = "Package: %s:%s is logged." % (pkg.cpv, pkg.repo,)
add_logs(session, log_msg, "info", config_id)
print("\n>>> Logging %s:%s\n" % (pkg.cpv, pkg.repo,))
+ build_msg = "BUILD: PASS"
+ qa_msg = "QA: PASS"
+ repoman_msg = "REPOMAN: PASS"
if build_log_dict['fail']:
- msg = "Package: %s Repo: %s FAILD Weblink http://foo.gg.oo/buildpackage/%s\n" % (pkg.cpv, pkg.repo, log_id,)
- else:
- msg = "Package: %s Repo: %s PASS Weblink http://foo.gg.oo/buildpackage/%s\n" % (pkg.cpv, pkg.repo, log_id,)
+ for error_id in build_log_dict['summary_error_list']:
+ if error_id == 1:
+ repoman_msg = "REPOMAN: FAILD"
+ elif error_id ==2:
+ qa_msg = "QA: FAILD"
+ else:
+ build_msg = "BUILD: FAILD"
+ msg = "Package: %s Repo: %s %s %s %s Weblink http://foo.gg.oo/buildpackage/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, log_id,)
send_irk(msg)
session.close
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-01-10 12:53 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-01-10 12:53 UTC (permalink / raw
To: gentoo-commits
commit: 80b205f448c096b57772d42706e5180f2c4849e0
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 10 12:54:58 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 10 12:54:58 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=80b205f4
change from checksum to mtime on Manifest
pym/tbc/db_mapping.py | 2 +-
pym/tbc/package.py | 83 ++++++++++++++++++++++++++-------------------------
pym/tbc/sqlquerys.py | 6 ++--
3 files changed, 46 insertions(+), 45 deletions(-)
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 82318a1..2636bb6 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -84,7 +84,7 @@ class Packages(Base):
CategoryId = Column('category_id', Integer, ForeignKey('categories.category_id'))
Package = Column('package',String(150))
RepoId = Column('repo_id', Integer, ForeignKey('repos.repo_id'))
- Checksum = Column('checksum', String(100))
+ Mtime = Column('mtime', DateTime, default=0)
Active = Column('active', Boolean, default=False)
TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
__tablename__ = 'packages'
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 091f75f..40de495 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -15,7 +15,7 @@ from tbc.qachecks import check_repoman
from tbc.build_log import check_repoman_full
from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
- get_package_metadata_sql, update_package_metadata, update_manifest_sql, \
+ get_package_metadata_sql, update_package_metadata, update_mtime_sql, \
get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \
get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info, \
get_ebuild_info_ebuild_id, get_ebuild_restrictions, add_old_package
@@ -206,7 +206,7 @@ class tbc_package(object):
package_metadataDict[package_id] = attDict
return package_metadataDict
- def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree):
+ def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree):
# Use packageDict to update the db
ebuild_id_list = add_new_ebuild_sql(self._session, packageDict)
@@ -224,7 +224,7 @@ class tbc_package(object):
update_package_metadata(self._session, package_metadataDict)
# update the cp manifest checksum
- update_manifest_sql(self._session, package_id, manifest_checksum_tree)
+ update_mtime_sql(self._session, package_id, manifest_mtime_tree)
# Get the best cpv for the configs and add it to config_cpv_listDict
PackageInfo, CategoryInfo, RepoInfo = get_package_info_from_package_id(self._session, package_id)
@@ -240,20 +240,17 @@ class tbc_package(object):
# Add the ebuild to the build jobs table if needed
self.add_new_build_job_db(ebuild_id_list, packageDict, config_cpv_listDict)
- def get_manifest_checksum_tree(self, pkgdir, cp, repo, mytree):
- # Get the cp manifest file checksum.
+ def get_manifest_mtime_tree(self, pkgdir, cp, repo, mytree):
+ # Get the cp manifest file mtime.
try:
- manifest_checksum_tree = portage.checksum.sha256hash(pkgdir + "/Manifest")[0]
+ mtime = os.path.getmtime(pkgdir + "/Manifest")
except:
- # Get the package list from the repo
- package_list_tree =self. _myportdb.cp_all(trees=mytree)
- if cp in package_list_tree:
- log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,)
- add_logs(self._session, log_msg, "error", self._config_id)
- log_msg = "C %s:%s ... Fail." % (cp, repo)
- add_logs(self._session, log_msg, "error", self._config_id)
- return "0"
- return manifest_checksum_tree
+ log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,)
+ add_logs(self._session, log_msg, "error", self._config_id)
+ log_msg = "C %s:%s ... Fail." % (cp, repo)
+ add_logs(self._session, log_msg, "error", self._config_id)
+ return False
+ return datetime.datetime.fromtimestamp(mtime).replace(microsecond=0)
def add_new_package_db(self, cp, repo):
# Add new categories package ebuild to tables package and ebuilds
@@ -267,8 +264,10 @@ class tbc_package(object):
mytree = []
mytree.append(repodir)
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
- manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
-
+ manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp, repo, mytree)
+ if not manifest_mtime_tree:
+ return None
+
package_id = add_new_package_sql(self._session, cp, repo)
# Check cp with repoman full
@@ -299,7 +298,7 @@ class tbc_package(object):
package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
packageDict[cpv]['new'] = True
- self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
+ self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
@@ -313,12 +312,28 @@ class tbc_package(object):
add_logs(self._session, log_msg, "info", self._config_id)
repodir = self._myportdb.getRepositoryPath(repo)
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
+ if not os.path.isdir(pkgdir):
+ old_ebuild_id_list = get_ebuild_id_list(self._session, package_id)
+ for ebuild_id in old_ebuild_id_list:
+ EbuildInfo = get_ebuild_info_ebuild_id(self._session, ebuild_id)
+ cpv = cp + "-" + EbuildInfo.Version
+ # R = remove ebuild
+ log_msg = "R %s:%s" % (cpv, repo,)
+ add_logs(self._session, log_msg, "info", self._config_id)
+ add_old_ebuild(self._session, old_ebuild_id_list)
+ add_old_package(self._session, package_id)
+ log_msg = "C %s:%s ... Done." % (cp, repo)
+ add_logs(self._session, log_msg, "info", self._config_id)
+ return None
+
mytree = []
mytree.append(repodir)
- manifest_checksum_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
+ manifest_mtime_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
+ if not manifest_mtime_tree:
+ return None
- # if we NOT have the same checksum in the db update the package
- if manifest_checksum_tree != PackageInfo.Checksum:
+ # if we NOT have the same mtime in the db update the package
+ if PackageInfo.Mtime is None or manifest_mtime_tree > PackageInfo.Mtime:
# U = Update
log_msg = "U %s:%s" % (cp, repo)
@@ -328,25 +343,11 @@ class tbc_package(object):
old_ebuild_id_list = []
ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
if ebuild_list_tree == []:
- if manifest_checksum_tree == "0":
- old_ebuild_id_list = get_ebuild_id_list(self._session, package_id)
- for ebuild_id in old_ebuild_id_list:
- EbuildInfo = get_ebuild_info_ebuild_id(self._session, ebuild_id)
- cpv = cp + "-" + EbuildInfo.Version
- # R = remove ebuild
- log_msg = "R %s:%s" % (cpv, repo,)
- add_logs(self._session, log_msg, "info", self._config_id)
- if not os.path.isdir(pkgdir):
- add_old_package(self._session, package_id)
- add_old_ebuild(self._session, old_ebuild_id_list)
- log_msg = "C %s:%s ... Done." % (cp, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
- else:
- log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
- add_logs(self._session, log_msg, "info", self._config_id)
- log_msg = "C %s:%s ... Fail." % (cp, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
- return
+ log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
+ add_logs(self._session, log_msg, "info", self._config_id)
+ log_msg = "C %s:%s ... Fail." % (cp, repo)
+ add_logs(self._session, log_msg, "info", self._config_id)
+ return None
# Check cp with repoman full
status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
@@ -403,7 +404,7 @@ class tbc_package(object):
del packageDict[cpv]
ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
new_ebuild_id_list.append(ebuild_id)
- self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_checksum_tree)
+ self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
add_logs(self._session, log_msg, "info", self._config_id)
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 7791e6e..e93c96d 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -466,7 +466,7 @@ def add_new_package_sql(session, cp, repo):
repo_id = RepoInfo.RepoId
CategoriesInfo = get_category_info(session, categories)
category_id = CategoriesInfo.CategoryId
- session.add(Packages(Package = package, CategoryId = category_id, RepoId = repo_id, Checksum = '0', Active = True))
+ session.add(Packages(Package = package, CategoryId = category_id, RepoId = repo_id, Active = True))
session.commit()
PackageInfo = get_package_info(session, categories, package, repo)
return PackageInfo.PackageId
@@ -512,9 +512,9 @@ def update_package_metadata(session, package_metadataDict):
EmailInfo = update_email_info(session, email)
PackagesEmailInfo = update_package_email_info(session, EmailInfo.EmailId, k)
-def update_manifest_sql(session, package_id, manifest_checksum_tree):
+def update_mtime_sql(session, package_id, manifest_mtime_tree):
PackagesInfo = session.query(Packages).filter_by(PackageId = package_id).one()
- PackagesInfo.Checksum = manifest_checksum_tree
+ PackagesInfo.Mtime = manifest_mtime_tree
session.commit()
def get_package_info_from_package_id(session, package_id):
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-01-18 20:26 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-01-18 20:26 UTC (permalink / raw
To: gentoo-commits
commit: 250f837e65c6e28381ca57ecbe97085974902bdf
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 18 20:28:04 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jan 18 20:28:04 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=250f837e
move the hash for the build log
pym/tbc/build_log.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 79dbe2a..7b3784a 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -28,7 +28,7 @@ from tbc.ConnectionManager import NewConnection
from tbc.sqlquerys import add_logs, get_config_id, get_ebuild_id_db, add_new_buildlog, \
get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \
add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \
- update_buildjobs_status, update_manifest_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
+ update_buildjobs_status, update_mtime_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
add_repoman_log
from sqlalchemy.orm import sessionmaker
@@ -107,7 +107,7 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
if ebuild_id_list is None:
log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
add_logs(session, log_msg, "info", config_id)
- update_manifest_sql(session, build_dict['package_id'], "0")
+ update_mtime_sql(session, build_dict['package_id'], 0)
else:
old_ebuild_id_list = []
for ebuild_id in ebuild_id_list:
@@ -293,7 +293,7 @@ def add_buildlog_main(settings, pkg, trees):
if error_log_list != []:
for log_line in error_log_list:
build_error = build_error + log_line
- log_hash.update(log_line.encode('utf-8'))
+ log_hash.update(build_error.encode('utf-8'))
build_log_dict['build_error'] = build_error
build_log_dict['log_hash'] = log_hash.hexdigest()
build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1]
@@ -322,7 +322,7 @@ def add_buildlog_main(settings, pkg, trees):
qa_msg = "QA: FAILD"
else:
build_msg = "BUILD: FAILD"
- msg = "Package: %s Repo: %s %s %s %s Weblink http://foo.gg.oo/buildpackage/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, log_id,)
+ msg = "Package: %s Repo: %s %s %s %s Weblink http://foo.gg.oo/new/logs/build/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, log_id,)
send_irk(msg)
session.close
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-01-25 22:48 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-01-25 22:48 UTC (permalink / raw
To: gentoo-commits
commit: 71d06a89176b48e1dbdbb971b703422334efc540
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Jan 25 22:48:27 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Jan 25 22:48:27 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=71d06a89
add support for logging part2
pym/tbc/build_log.py | 2 +-
pym/tbc/package.py | 65 +++++++++++++++++++++++++++-------------------------
pym/tbc/sqlquerys.py | 2 +-
3 files changed, 36 insertions(+), 33 deletions(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 70cd091..a16cdc6 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -304,7 +304,7 @@ def add_buildlog_main(settings, pkg, trees):
build_log_dict['log_hash'] = log_hash.hexdigest()
build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1]
log_msg = "Logfile name: %s" % (settings.get("PORTAGE_LOG_FILE"),)
- write_log(session, log_msg, "info", config_id, 'build_log.add_buildlog_main')
+ write_log(session, log_msg, "info", config_id, 'build_log.add_buildlog_main')
build_log_dict['emerge_info'] = get_emerge_info_id(settings, trees, session, config_id)
log_id = add_new_buildlog(session, build_dict, build_log_dict)
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 72108a2..65a0f40 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -7,6 +7,7 @@ import hashlib
import os
import git
import portage
+import datetime
from portage.xml.metadata import MetaDataXML
from tbc.flags import tbc_use_flags
from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
@@ -83,6 +84,7 @@ class tbc_package(object):
if ebuild_auxdb_list[i] == '':
ebuild_auxdb_list[i] = ''
return ebuild_auxdb_list
+
def get_git_log_ebuild(self, repodir, ebuild_file):
git_log_ebuild = ''
g = git.Git(repodir)
@@ -105,11 +107,12 @@ class tbc_package(object):
except:
ebuild_version_checksum_tree = "0"
log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,)
- write_log(session, log_msg, "warning", config_id, 'packages.get_packageDict')
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_packageDict')
log_msg = "C %s:%s ... Fail." % (cpv, repo)
- write_log(session, log_msg, "info", config_id, 'packages.get_packageDict')
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.get_packageDict')
git_commit = '0'
else:
+ repodir =self._ myportdb.getRepositoryPath(repo)
git_commit = self.get_git_log_ebuild(repodir, ebuild_file)
# Get the ebuild metadata
@@ -119,7 +122,7 @@ class tbc_package(object):
# so it can be updated next time we update the db
if not ebuild_version_metadata_tree:
log_msg = " QA: %s have broken metadata on repo %s" % (cpv, repo)
- write_log(session, log_msg, "warning", config_id, 'packages.get_packageDict')
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_packageDict')
ebuild_version_metadata_tree = ['','','','','','','','','','','','','','','','','','','','','','','','','']
ebuild_version_checksum_tree = '0'
@@ -168,17 +171,17 @@ class tbc_package(object):
# B = Build cpv use-flags config
# FIXME log_msg need a fix to log the use flags corect.
log_msg = "B %s:%s USE: %s Setup: %s" % (k, v['repo'], use_flagsDict, setup_id,)
- write_log(session, log_msg, "info", config_id, 'packages.add_new_build_job_db')
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_build_job_db')
i = i +1
- def get_git_log_pkg_text(self, repodir, cp):
+ def get_git_changelog_text(self, repodir, cp):
n = '5'
git_log_pkg = ''
g = git.Git(repodir)
git_log_pkg = g.log('-n ' + n, '--grep=' + cp)
return git_log_pkg
- def get_package_metadataDict(self, pkgdir, package_id):
+ def get_package_metadataDict(self, pkgdir, repodir, package_id, cp):
# Make package_metadataDict
attDict = {}
package_metadataDict = {}
@@ -189,7 +192,7 @@ class tbc_package(object):
pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
except:
log_msg = "Metadata file %s is missing or has errors" % (pkgdir + "/metadata.xml")
- write_log(session, log_msg, "warning", config_id, 'packages.get_package_metadataDict')
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_package_metadataDict')
else:
tmp_herds = pkg_md.herds()
if tmp_herds != ():
@@ -201,8 +204,8 @@ class tbc_package(object):
attDict['metadata_xml_email'] = md_email_list
else:
log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
- write_log(session, log_msg, "warning", config_id, 'packages.get_package_metadataDict')
- attDict['git_log_pkg_text'] = self.get_git_log_pkg_text(repodir, cp)
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_package_metadataDict')
+ attDict['git_changlog'] = self.get_git_changelog_text(repodir, cp)
attDict['metadata_xml_descriptions'] = ''
package_metadataDict[package_id] = attDict
return package_metadataDict
@@ -247,9 +250,9 @@ class tbc_package(object):
mtime = os.path.getmtime(pkgdir + "/Manifest")
except:
log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,)
- write_log(session, log_msg, "warning", config_id, 'packages.get_manifest_mtime_tree')
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_manifest_mtime_tree')
log_msg = "C %s:%s ... Fail." % (cp, repo)
- write_log(session, log_msg, "warning", config_id, 'packages.get_manifest_mtime_tree')
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_manifest_mtime_tree')
return False
return datetime.datetime.fromtimestamp(mtime).replace(microsecond=0)
@@ -258,9 +261,9 @@ class tbc_package(object):
# C = Checking
# N = New Package
log_msg = "C %s:%s" % (cp, repo)
- write_log(session, log_msg, "info", config_id, 'packages.add_new_package_db')
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_package_db')
log_msg = "N %s:%s" % (cp, repo)
- write_log(session, log_msg, "info", config_id, 'packages.add_new_package_db')
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_package_db')
repodir = self._myportdb.getRepositoryPath(repo)
mytree = []
mytree.append(repodir)
@@ -275,16 +278,16 @@ class tbc_package(object):
status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
if status:
log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
- write_log(session, log_msg, "warning", config_id, 'packages.add_new_package_db')
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.add_new_package_db')
- package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
+ package_metadataDict = self.get_package_metadataDict(pkgdir, repodir, package_id, cp)
# Get the ebuild list for cp
ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
if ebuild_list_tree == []:
log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
- write_log(session, log_msg, "error", config_id, 'packages.add_new_package_db')
+ write_log(self._session, log_msg, "error", self._config_id, 'packages.add_new_package_db')
log_msg = "C %s:%s ... Fail." % (cp, repo)
- write_log(session, log_msg, "warning", config_id, 'packages.add_new_package_db')
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.add_new_package_db')
return None
# Make the needed packageDict with ebuild infos so we can add it later to the db.
@@ -301,7 +304,7 @@ class tbc_package(object):
packageDict[cpv]['new'] = True
self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
- write_log(session, log_msg, "info", config_id, 'packages.add_new_package_db')
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_package_db')
def update_package_db(self, package_id):
# Update the categories and package with new info
@@ -310,7 +313,7 @@ class tbc_package(object):
cp = CategoryInfo.Category + '/' + PackageInfo.Package
repo = RepoInfo.Repo
log_msg = "C %s:%s" % (cp, repo)
- write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
repodir = self._myportdb.getRepositoryPath(repo)
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
if not os.path.isdir(pkgdir):
@@ -320,16 +323,16 @@ class tbc_package(object):
cpv = cp + "-" + EbuildInfo.Version
# R = remove ebuild
log_msg = "R %s:%s" % (cpv, repo,)
- write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
add_old_ebuild(self._session, old_ebuild_id_list)
add_old_package(self._session, package_id)
log_msg = "C %s:%s ... Done." % (cp, repo)
- write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
return None
mytree = []
mytree.append(repodir)
- manifest_mtime_tree = self.get_manifest_checksum_tree(pkgdir, cp, repo, mytree)
+ manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp, repo, mytree)
if not manifest_mtime_tree:
return None
@@ -338,24 +341,24 @@ class tbc_package(object):
# U = Update
log_msg = "U %s:%s" % (cp, repo)
- write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
# Get the ebuild list for cp
old_ebuild_id_list = []
ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
if ebuild_list_tree == []:
log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
- write_log(session, log_msg, "error", config_id, 'packages.update_package_db')
+ write_log(self._session, log_msg, "error", self._config_id, 'packages.update_package_db')
log_msg = "C %s:%s ... Fail." % (cp, repo)
- write_log(session, log_msg, "warning", config_id, 'packages.update_package_db')
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
return None
# Check cp with repoman full
status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
if status:
log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
- write_log(session, log_msg, "warning", config_id, 'packages.update_package_db')
- package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
+ package_metadataDict = self.get_package_metadataDict(pkgdir, repodir, package_id, cp)
packageDict ={}
new_ebuild_id_list = []
for cpv in sorted(ebuild_list_tree):
@@ -383,7 +386,7 @@ class tbc_package(object):
ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id)
for ebuild_id in ebuilds_id:
log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,)
- write_log(session, log_msg, "warning", config_id, 'packages.update_package_db')
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
dupe_ebuild_id_list.append(ebuild_id)
add_old_ebuild(self._session, dupe_ebuild_id_list)
ebuild_version_manifest_checksum_db = None
@@ -394,12 +397,12 @@ class tbc_package(object):
if ebuild_version_manifest_checksum_db is None:
# N = New ebuild
log_msg = "N %s:%s" % (cpv, repo,)
- write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
packageDict[cpv]['new'] = True
elif ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
# U = Updated ebuild
log_msg = "U %s:%s" % (cpv, repo,)
- write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
else:
# Remove cpv from packageDict and add ebuild to new ebuils list
del packageDict[cpv]
@@ -408,4 +411,4 @@ class tbc_package(object):
self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
- write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 5100bec..e43b8fc 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -422,7 +422,7 @@ def add_new_ebuild_sql(session, packageDict):
except (MultipleResultsFound) as e:
# FIXME
sys.exit()
- session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, Revision = v['ebuild_version_revision_tree'], Descriptions = v['ebuild_version_descriptions_tree']))
+ session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, Commit = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
session.commit()
ebuild_id_list.append(EbuildInfo.EbuildId)
restrictions = []
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-01-26 20:29 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-01-26 20:29 UTC (permalink / raw
To: gentoo-commits
commit: 85d70df173a81e8257172d39be1ae971be54422b
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 26 20:30:38 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jan 26 20:30:38 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=85d70df1
check all ebuilds in package instead of only check Manifest
pym/tbc/build_log.py | 5 +-
pym/tbc/db_mapping.py | 1 -
pym/tbc/package.py | 151 +++++++++++++++++++++-----------------------------
pym/tbc/sqlquerys.py | 5 --
pym/tbc/sync.py | 20 +++++--
5 files changed, 80 insertions(+), 102 deletions(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index a16cdc6..ec4aaa7 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -29,7 +29,7 @@ from tbc.ConnectionManager import NewConnection
from tbc.sqlquerys import get_config_id, get_ebuild_id_db, add_new_buildlog, \
get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \
add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \
- update_buildjobs_status, update_mtime_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
+ update_buildjobs_status, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
add_repoman_log
from tbc.log import write_log
@@ -112,8 +112,7 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
if status:
if ebuild_id_list is None:
log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
- add_logs(session, log_msg, "info", config_id)
- update_mtime_sql(session, build_dict['package_id'], 0)
+ add_logs(session, log_msg, "error", config_id)
else:
old_ebuild_id_list = []
for ebuild_id in ebuild_id_list:
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 2636bb6..d9fcf25 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -84,7 +84,6 @@ class Packages(Base):
CategoryId = Column('category_id', Integer, ForeignKey('categories.category_id'))
Package = Column('package',String(150))
RepoId = Column('repo_id', Integer, ForeignKey('repos.repo_id'))
- Mtime = Column('mtime', DateTime, default=0)
Active = Column('active', Boolean, default=False)
TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
__tablename__ = 'packages'
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 65a0f40..68574cf 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -17,7 +17,7 @@ from tbc.build_log import check_repoman_full
from tbc.log import write_log
from tbc.sqlquerys import get_package_info, get_config_info, \
add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
- get_package_metadata_sql, update_package_metadata, update_mtime_sql, \
+ get_package_metadata_sql, update_package_metadata, \
get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \
get_ebuild_checksums, get_ebuild_id_db, get_configmetadata_info, get_setup_info, \
get_ebuild_info_ebuild_id, get_ebuild_restrictions, add_old_package
@@ -210,7 +210,7 @@ class tbc_package(object):
package_metadataDict[package_id] = attDict
return package_metadataDict
- def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree):
+ def add_package(self, packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list):
# Use packageDict to update the db
ebuild_id_list = add_new_ebuild_sql(self._session, packageDict)
@@ -227,9 +227,6 @@ class tbc_package(object):
# update package metadata
update_package_metadata(self._session, package_metadataDict)
- # update the cp manifest checksum
- update_mtime_sql(self._session, package_id, manifest_mtime_tree)
-
# Get the best cpv for the configs and add it to config_cpv_listDict
PackageInfo, CategoryInfo, RepoInfo = get_package_info_from_package_id(self._session, package_id)
cp = CategoryInfo.Category + '/' + PackageInfo.Package
@@ -244,18 +241,6 @@ class tbc_package(object):
# Add the ebuild to the build jobs table if needed
self.add_new_build_job_db(ebuild_id_list, packageDict, config_cpv_listDict)
- def get_manifest_mtime_tree(self, pkgdir, cp, repo, mytree):
- # Get the cp manifest file mtime.
- try:
- mtime = os.path.getmtime(pkgdir + "/Manifest")
- except:
- log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,)
- write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_manifest_mtime_tree')
- log_msg = "C %s:%s ... Fail." % (cp, repo)
- write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_manifest_mtime_tree')
- return False
- return datetime.datetime.fromtimestamp(mtime).replace(microsecond=0)
-
def add_new_package_db(self, cp, repo):
# Add new categories package ebuild to tables package and ebuilds
# C = Checking
@@ -268,9 +253,6 @@ class tbc_package(object):
mytree = []
mytree.append(repodir)
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
- manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp, repo, mytree)
- if not manifest_mtime_tree:
- return None
package_id = add_new_package_sql(self._session, cp, repo)
@@ -332,83 +314,78 @@ class tbc_package(object):
mytree = []
mytree.append(repodir)
- manifest_mtime_tree = self.get_manifest_mtime_tree(pkgdir, cp, repo, mytree)
- if not manifest_mtime_tree:
+
+ # Get the ebuild list for cp
+ old_ebuild_id_list = []
+ ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
+ if ebuild_list_tree == []:
+ log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
+ write_log(self._session, log_msg, "error", self._config_id, 'packages.update_package_db')
+ log_msg = "C %s:%s ... Fail." % (cp, repo)
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
return None
- # if we NOT have the same mtime in the db update the package
- if PackageInfo.Mtime is None or manifest_mtime_tree > PackageInfo.Mtime:
+ package_metadataDict = self.get_package_metadataDict(pkgdir, repodir, package_id, cp)
+ packageDict ={}
+ new_ebuild_id_list = []
+ package_updated = False
+ for cpv in sorted(ebuild_list_tree):
- # U = Update
- log_msg = "U %s:%s" % (cp, repo)
- write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
+ # split out ebuild version
+ ebuild_version_tree = portage.versions.cpv_getversion(cpv)
- # Get the ebuild list for cp
- old_ebuild_id_list = []
- ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
- if ebuild_list_tree == []:
- log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
- write_log(self._session, log_msg, "error", self._config_id, 'packages.update_package_db')
- log_msg = "C %s:%s ... Fail." % (cp, repo)
- write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
- return None
+ # Get packageDict for cpv
+ packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
+
+ # take package descriptions from the ebuilds
+ if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
+ package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
+
+ # Get the checksum of the ebuild in tree and db
+ ebuild_version_checksum_tree = packageDict[cpv]['checksum']
+ checksums_db, fail = get_ebuild_checksums(self._session, package_id, ebuild_version_tree)
+
+ # check if we have dupes of the checksum from db
+ if checksums_db is None:
+ ebuild_version_manifest_checksum_db = None
+ elif fail:
+ dupe_ebuild_id_list = []
+ for checksum in checksums_db:
+ ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id)
+ for ebuild_id in ebuilds_id:
+ log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,)
+ write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
+ dupe_ebuild_id_list.append(ebuild_id)
+ add_old_ebuild(self._session, dupe_ebuild_id_list)
+ ebuild_version_manifest_checksum_db = None
+ else:
+ ebuild_version_manifest_checksum_db = checksums_db
+
+ # Check if the checksum have change
+ if ebuild_version_manifest_checksum_db is None:
+ # N = New ebuild
+ log_msg = "N %s:%s" % (cpv, repo,)
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
+ packageDict[cpv]['new'] = True
+ package_updated = True
+ elif ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
+ # U = Updated ebuild
+ log_msg = "U %s:%s" % (cpv, repo,)
+ write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
+ package_updated = True
+ else:
+ # Remove cpv from packageDict and add ebuild to new ebuils list
+ del packageDict[cpv]
+ ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
+ new_ebuild_id_list.append(ebuild_id)
+ self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
+ if package_updated:
# Check cp with repoman full
status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
if status:
log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
- package_metadataDict = self.get_package_metadataDict(pkgdir, repodir, package_id, cp)
- packageDict ={}
- new_ebuild_id_list = []
- for cpv in sorted(ebuild_list_tree):
-
- # split out ebuild version
- ebuild_version_tree = portage.versions.cpv_getversion(cpv)
-
- # Get packageDict for cpv
- packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
-
- # take package descriptions from the ebuilds
- if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
- package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
-
- # Get the checksum of the ebuild in tree and db
- ebuild_version_checksum_tree = packageDict[cpv]['checksum']
- checksums_db, fail = get_ebuild_checksums(self._session, package_id, ebuild_version_tree)
-
- # check if we have dupes of the checksum from db
- if checksums_db is None:
- ebuild_version_manifest_checksum_db = None
- elif fail:
- dupe_ebuild_id_list = []
- for checksum in checksums_db:
- ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id)
- for ebuild_id in ebuilds_id:
- log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,)
- write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
- dupe_ebuild_id_list.append(ebuild_id)
- add_old_ebuild(self._session, dupe_ebuild_id_list)
- ebuild_version_manifest_checksum_db = None
- else:
- ebuild_version_manifest_checksum_db = checksums_db
-
- # Check if the checksum have change
- if ebuild_version_manifest_checksum_db is None:
- # N = New ebuild
- log_msg = "N %s:%s" % (cpv, repo,)
- write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
- packageDict[cpv]['new'] = True
- elif ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
- # U = Updated ebuild
- log_msg = "U %s:%s" % (cpv, repo,)
- write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
- else:
- # Remove cpv from packageDict and add ebuild to new ebuils list
- del packageDict[cpv]
- ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
- new_ebuild_id_list.append(ebuild_id)
- self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index e43b8fc..4161461 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -515,11 +515,6 @@ def update_package_metadata(session, package_metadataDict):
EmailInfo = update_email_info(session, email)
PackagesEmailInfo = update_package_email_info(session, EmailInfo.EmailId, k)
-def update_mtime_sql(session, package_id, manifest_mtime_tree):
- PackagesInfo = session.query(Packages).filter_by(PackageId = package_id).one()
- PackagesInfo.Mtime = manifest_mtime_tree
- session.commit()
-
def get_package_info_from_package_id(session, package_id):
PackageInfo = session.query(Packages).filter_by(PackageId = package_id).one()
CategoryInfo = session.query(Categories).filter_by(CategoryId = PackageInfo.CategoryId).one()
diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 1732f1f..2d22e74 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -70,9 +70,10 @@ def git_sync_main(session):
except:
pass
- # check git diffs witch Manifests get updated and pass that to a dict
+ # check git diffs witch get updated and pass that to a dict
# fetch and merge the repo
repo_cp_dict = {}
+ search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts',]
for repo_dir in git_repos_list(myportdb):
reponame = myportdb.getRepositoryName(repo_dir)
repo = git.Repo(repo_dir)
@@ -82,14 +83,21 @@ def git_sync_main(session):
if not repouptodate:
cp_list = []
attr = {}
- # We check for Manifest changes and add the package to a list
+ # We check for dir changes and add the package to a list
repo_diff = repo.git.diff('origin', '--name-only')
- write_log(session, 'Git diff: %s' % (repo_diff,), "debug", config_id, 'sync.git_sync_main')
+ write_log(session, 'Git dir diff:\n%s' % (repo_diff,), "debug", config_id, 'sync.git_sync_main')
for diff_line in repo_diff.splitlines():
- if re.search("Manifest$", diff_line):
- cp = re.sub('/Manifest', '', diff_line)
- cp_list.append(cp)
+ find_search = True
+ for search_line in search_list:
+ if re.search(search_line, diff_line):
+ find_search = False
+ if find_search:
+ splited_diff_line = re.split('/', diff_line)
+ cp = splited_diff_line[0] + '/' + splited_diff_line[1]
+ if not cp in cp_list:
+ cp_list.append(cp)
attr['cp_list'] = cp_list
+ write_log(session, 'Git CP Diff: %s' % (cp_list,), "debug", config_id, 'sync.git_sync_main')
repo_cp_dict[reponame] = attr
git_merge(repo, info_list[0])
else:
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-01-26 23:16 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-01-26 23:16 UTC (permalink / raw
To: gentoo-commits
commit: 4b895c761e79db78b1c3640d5d053c93fdd98277
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 26 23:17:20 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jan 26 23:17:20 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4b895c76
fix for Dups of checksums
pym/tbc/build_log.py | 6 +++---
pym/tbc/package.py | 2 +-
pym/tbc/sqlquerys.py | 6 +++---
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index d4fa0ef..b194561 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -108,16 +108,16 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package
ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir+ "/" + package + "-" + ebuild_version + ".ebuild")[0]
build_dict['checksum'] = ebuild_version_checksum_tree
- ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id'])
+ ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id'], build_dict['ebuild_version'])
if status:
if ebuild_id_list is None:
log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
- add_logs(session, log_msg, "error", config_id)
+ write_log(session, log_msg, "error", config_id, 'build_log.get_build_dict_db')
else:
old_ebuild_id_list = []
for ebuild_id in ebuild_id_list:
log_msg = "%s:%s:%s Dups of checksums" % (pkg.cpv, repo, ebuild_id,)
- add_logs(session, log_msg, "error", config_id)
+ write_log(session, log_msg, "error", config_id, 'build_log.get_build_dict_db')
old_ebuild_id_list.append(ebuild_id)
add_old_ebuild(session, old_ebuild_id_list)
return
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index bb797de..60e0de5 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -351,7 +351,7 @@ class tbc_package(object):
elif fail:
dupe_ebuild_id_list = []
for checksum in checksums_db:
- ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id)
+ ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id, ebuild_version_tree)
for ebuild_id in ebuilds_id:
log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,)
write_log(self._session, log_msg, "warning", self._config_id, 'packages.update_package_db')
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 4161461..2fdb7f1 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -545,13 +545,13 @@ def get_ebuild_checksums(session, package_id, ebuild_version):
return ebuild_checksum_list, True
return EbuildInfo.Checksum, False
-def get_ebuild_id_db(session, checksum, package_id):
+def get_ebuild_id_db(session, checksum, package_id, ebuild_version):
try:
- EbuildInfos = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Checksum = checksum).one()
+ EbuildInfos = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Checksum = checksum).filter_by(Version = ebuild_version).filter_by(Active = True).one()
except NoResultFound as e:
return None, True
except MultipleResultsFound as e:
- EbuildInfos = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Checksum = checksum).all()
+ EbuildInfos = session.query(Ebuilds).filter_by(PackageId = package_id).filter_by(Checksum = checksum).filter_by(Version = ebuild_version).filter_by(Active = True).all()
ebuilds_id = []
for EbuildInfo in EbuildInfos:
ebuilds_id.append(EbuildInfo.EbuildId)
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-01-27 5:21 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-01-27 5:21 UTC (permalink / raw
To: gentoo-commits
commit: cc16d58199d60e5f65b86c3f8a59063e52c8cdae
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Jan 27 05:22:11 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Jan 27 05:22:11 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=cc16d581
check all ebuilds in package instead of only check Manifest small fix
pym/tbc/package.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 60e0de5..cf711ff 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -284,7 +284,7 @@ class tbc_package(object):
package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
packageDict[cpv]['new'] = True
- self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
+ self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list)
log_msg = "C %s:%s ... Done." % (cp, repo)
write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_package_db')
@@ -378,7 +378,7 @@ class tbc_package(object):
del packageDict[cpv]
ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
new_ebuild_id_list.append(ebuild_id)
- self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
+ self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list)
if package_updated:
# Check cp with repoman full
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-01-31 13:39 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-01-31 13:39 UTC (permalink / raw
To: gentoo-commits
commit: 28dc9209b9e9787a1c9dc968f68ad89051a699f6
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 31 13:40:04 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 31 13:40:04 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=28dc9209
small fixes
pym/tbc/package.py | 2 +-
pym/tbc/sqlquerys.py | 4 ++--
pym/tbc/sync.py | 20 ++++++++++----------
3 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index cf711ff..510a5d2 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -376,7 +376,7 @@ class tbc_package(object):
else:
# Remove cpv from packageDict and add ebuild to new ebuils list
del packageDict[cpv]
- ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id)
+ ebuild_id , status = get_ebuild_id_db(self._session, ebuild_version_checksum_tree, package_id, ebuild_version_tree)
new_ebuild_id_list.append(ebuild_id)
self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list)
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 2fdb7f1..9ada5dd 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -504,10 +504,10 @@ def update_package_metadata(session, package_metadataDict):
try:
PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = k).one()
except NoResultFound as e:
- session.add(PackagesMetadata(PackageId = k, Changlog = v['changelog_text'], Descriptions = v['metadata_xml_descriptions']))
+ session.add(PackagesMetadata(PackageId = k, Gitlog = v['git_changlog'], Descriptions = v['metadata_xml_descriptions']))
session.commit()
else:
- PackagesMetadataInfo.Changlog = v['changelog_text']
+ PackagesMetadataInfo.Gitlog = v['git_changlog']
PackagesMetadataInfo.Descriptions = v['metadata_xml_descriptions']
session.commit()
if v['metadata_xml_email']:
diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 2d22e74..52d84ab 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -73,7 +73,7 @@ def git_sync_main(session):
# check git diffs witch get updated and pass that to a dict
# fetch and merge the repo
repo_cp_dict = {}
- search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts',]
+ search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts', '^skel.ebuild',]
for repo_dir in git_repos_list(myportdb):
reponame = myportdb.getRepositoryName(repo_dir)
repo = git.Repo(repo_dir)
@@ -87,15 +87,15 @@ def git_sync_main(session):
repo_diff = repo.git.diff('origin', '--name-only')
write_log(session, 'Git dir diff:\n%s' % (repo_diff,), "debug", config_id, 'sync.git_sync_main')
for diff_line in repo_diff.splitlines():
- find_search = True
- for search_line in search_list:
- if re.search(search_line, diff_line):
- find_search = False
- if find_search:
- splited_diff_line = re.split('/', diff_line)
- cp = splited_diff_line[0] + '/' + splited_diff_line[1]
- if not cp in cp_list:
- cp_list.append(cp)
+ find_search = True
+ for search_line in search_list:
+ if re.search(search_line, diff_line):
+ find_search = False
+ if find_search:
+ splited_diff_line = re.split('/', diff_line)
+ cp = splited_diff_line[0] + '/' + splited_diff_line[1]
+ if not cp in cp_list:
+ cp_list.append(cp)
attr['cp_list'] = cp_list
write_log(session, 'Git CP Diff: %s' % (cp_list,), "debug", config_id, 'sync.git_sync_main')
repo_cp_dict[reponame] = attr
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-02-02 23:13 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-02-02 23:13 UTC (permalink / raw
To: gentoo-commits
commit: d1cf1a085c322865694c738c23e655421b9364ce
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 2 23:14:54 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Feb 2 23:14:54 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d1cf1a08
split build error from repoman/qa
pym/tbc/build_log.py | 4 +---
pym/tbc/sync.py | 2 +-
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index b194561..c70dc76 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -241,16 +241,14 @@ def get_buildlog_info(session, settings, pkg, build_dict, config_id):
pkgdir = myportdb.getRepositoryPath(build_dict['repo']) + "/" + categories + "/" + package
repoman_error_list = check_repoman_full(session, pkgdir, build_dict['package_id'], config_id, build_dict['cpv'])
build_log_dict = {}
+ error_search_line = "^ \\* ERROR: "
build_log_dict['fail'] = False
if repoman_error_list:
sum_build_log_list.append(1) # repoman = 1
- build_log_dict['fail'] = True
if qa_error_list != []:
sum_build_log_list.append(2) # qa = 2
- build_log_dict['fail'] = True
else:
qa_error_list = False
- error_search_line = "^ \\* ERROR: "
for error_log_line in error_log_list:
if re.search(error_search_line, error_log_line):
build_log_dict['fail'] = True
diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 52d84ab..59560f0 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -73,7 +73,7 @@ def git_sync_main(session):
# check git diffs witch get updated and pass that to a dict
# fetch and merge the repo
repo_cp_dict = {}
- search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts', '^skel.ebuild',]
+ search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts', '^skel.', ]
for repo_dir in git_repos_list(myportdb):
reponame = myportdb.getRepositoryName(repo_dir)
repo = git.Repo(repo_dir)
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-02-10 20:24 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-02-10 20:24 UTC (permalink / raw
To: gentoo-commits
commit: 14b0918ae227f4135477946c8e6c9d9919b56fbf
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Feb 10 20:25:25 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Feb 10 20:25:25 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=14b0918a
fix so we mark any error as fail and some small fixes
pym/tbc/build_log.py | 11 +++++++----
pym/tbc/db_mapping.py | 6 ++++++
pym/tbc/package.py | 2 +-
pym/tbc/sqlquerys.py | 8 ++++++--
4 files changed, 20 insertions(+), 7 deletions(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index c70dc76..8b6ef0b 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -30,7 +30,7 @@ from tbc.sqlquerys import get_config_id, get_ebuild_id_db, add_new_buildlog, \
get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \
add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \
update_buildjobs_status, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
- add_repoman_log
+ add_repoman_log, get_tbc_config
from tbc.log import write_log
from sqlalchemy.orm import sessionmaker
@@ -224,13 +224,13 @@ def get_buildlog_info(session, settings, pkg, build_dict, config_id):
for k, v in sorted(hilight_dict.items()):
if v['startline'] == v['endline']:
error_log_list.append(logfile_text_dict[k])
- if v['hilight_css_id'] == 3: # qa = 3 and 4
+ if v['hilight_css_id'] == 3: # qa = 3
qa_error_list.append(logfile_text_dict[k])
else:
i = k
while i != (v['endline'] + 1):
error_log_list.append(logfile_text_dict[i])
- if v['hilight_css_id'] == 3: # qa = 3 and 4
+ if v['hilight_css_id'] == 3: # qa = 3
qa_error_list.append(logfile_text_dict[i])
i = i +1
@@ -245,8 +245,10 @@ def get_buildlog_info(session, settings, pkg, build_dict, config_id):
build_log_dict['fail'] = False
if repoman_error_list:
sum_build_log_list.append(1) # repoman = 1
+ build_log_dict['fail'] = True
if qa_error_list != []:
sum_build_log_list.append(2) # qa = 2
+ build_log_dict['fail'] = True
else:
qa_error_list = False
for error_log_line in error_log_list:
@@ -324,7 +326,8 @@ def add_buildlog_main(settings, pkg, trees):
qa_msg = "QA: FAILD"
else:
build_msg = "BUILD: FAILD"
- msg = "Package: %s Repo: %s %s %s %s Weblink http://77.110.8.67/new/logs/build/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, log_id,)
+ tbc_config = get_tbc_config(session)
+ msg = "Package: %s Repo: %s %s %s %s Weblink http://%s/new/logs/build/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, tbc_config.WebIker, log_id,)
write_log(session, msg, "info", config_id, 'build_log.add_buildlog_main')
send_irk(msg)
session.close
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index b0ab23f..9be0d0a 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -287,3 +287,9 @@ class EbuildsMetadata(Base):
New = Column('new', Boolean, default=False)
Descriptions = Column('descriptions', Text)
__tablename__ = 'ebuilds_metadata'
+
+class TbcConfig(Base):
+ Id = Column('id', Integer, primary_key=True)
+ WebIker = Column('webinker', String)
+ WebBug = Column('webbug', String)
+ __tablename__ = 'tbc_config'
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 510a5d2..c069c7e 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -136,7 +136,7 @@ class tbc_package(object):
attDict['ebuild_version_metadata_tree'] = ebuild_version_metadata_tree
#attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
attDict['git_commit'] = git_commit
- attDict['New'] = False
+ attDict['new'] = False
attDict['ebuild_version_descriptions_tree'] = ebuild_version_metadata_tree[7]
return attDict
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 5a8a183..f9f14a2 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -9,12 +9,16 @@ from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Pack
HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \
BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \
Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepoman, CategoriesMetadata, \
- PackagesRepoman, BuildLogsQa
+ PackagesRepoman, BuildLogsQa, TbcConfig
from tbc.log import write_log
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
from sqlalchemy import and_, or_
# Guest Functions
+def get_tbc_config(session):
+ TbcConfigInfo = session.query(TbcConfig).one()
+ return TbcConfigInfo
+
def get_config_id(session, setup, host):
SetupInfo = session.query(Setups).filter_by(Setup = setup).one()
ConfigInfo = session.query(Configs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Hostname = host).one()
@@ -419,7 +423,7 @@ def add_new_ebuild_sql(session, packageDict):
except (MultipleResultsFound) as e:
# FIXME
sys.exit()
- session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, Commit = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
+ session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Commit = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
session.commit()
ebuild_id_list.append(EbuildInfo.EbuildId)
restrictions = []
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-02-20 12:12 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-02-20 12:12 UTC (permalink / raw
To: gentoo-commits
commit: 426c02c2521b862ed7876c3fdb3018b34f843224
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 20 12:12:50 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Feb 20 12:12:50 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=426c02c2
Remove the profile/config and some prints
pym/tbc/flags.py | 3 ---
pym/tbc/sync.py | 14 --------------
2 files changed, 17 deletions(-)
diff --git a/pym/tbc/flags.py b/pym/tbc/flags.py
index eea48f3..82b7d42 100644
--- a/pym/tbc/flags.py
+++ b/pym/tbc/flags.py
@@ -117,7 +117,6 @@ class tbc_use_flags(object):
# reset cpv filter
self._mysettings.reset()
self._mysettings.lock()
- print(usemask)
return use, use_expand_hidden, usemask, useforce
def get_all_cpv_use_looked(self):
@@ -218,7 +217,6 @@ class tbc_use_flags(object):
use_flagsDict[x] = True
for x in use_disable:
use_flagsDict[x] = False
- print("use_flagsDict", use_flagsDict)
for k, v in use_flagsDict.items():
if build_use_flags_dict[k] != v:
if build_use_flags_dict[k]:
@@ -227,5 +225,4 @@ class tbc_use_flags(object):
build_use_flags_list.append("-" + k)
if build_use_flags_list == []:
build_use_flags_list = None
- print(build_use_flags_list)
return build_use_flags_list
diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 59560f0..7063bfb 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -63,12 +63,6 @@ def git_sync_main(session):
GuestBusy = False
else:
time.sleep(60)
- #remove the needed base profile clone
- try:
- os.remove(mysettings['PORTDIR'] + "/profiles/config/parent")
- os.rmdir(mysettings['PORTDIR'] + "/profiles/config")
- except:
- pass
# check git diffs witch get updated and pass that to a dict
# fetch and merge the repo
@@ -105,14 +99,6 @@ def git_sync_main(session):
write_log(session, log_msg, "info", config_id, 'sync.git_sync_main')
log_msg = "Checking repo %s Done" % (reponame)
write_log(session, log_msg, "info", config_id, 'sync.git_sync_main')
- # Need to add a clone of profiles/base for reading the tree
- try:
- os.mkdir(mysettings['PORTDIR'] + "/profiles/config", 0o777)
- with open(mysettings['PORTDIR'] + "/profiles/config/parent", "w") as f:
- f.write("../base\n")
- f.close()
- except:
- pass
log_msg = "Repo sync ... Done."
write_log(session, log_msg, "info", config_id, 'sync.git_sync_main')
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-02-22 17:08 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-02-22 17:08 UTC (permalink / raw
To: gentoo-commits
commit: 5e5291855b1206e942125bc8fd3bca360c7fdcb5
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 22 17:09:41 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb 22 17:09:41 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=5e529185
fix typo in remove_old_cpv_main()
pym/tbc/old_cpv.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/pym/tbc/old_cpv.py b/pym/tbc/old_cpv.py
index 027f171..cc80428 100644
--- a/pym/tbc/old_cpv.py
+++ b/pym/tbc/old_cpv.py
@@ -38,7 +38,7 @@ def remove_old_ebuilds(package_id, config_id, tbc_settings, cp):
Session.remove()
def remove_old_cpv_main(session, config_id):
- today = datetime.datetime.utcnow
+ today = datetime.datetime.utcnow()
tbc_settings = read_config_settings()
# Use all cores when multiprocessing
#pool_cores = multiprocessing.cpu_count()
@@ -65,5 +65,5 @@ def remove_old_cpv_main(session, config_id):
#close and join the multiprocessing pools
# pool.close()
# pool.join()
- if not get_package_list_info(session, CategoryId = CategoryInfo.CategoryId):
- add_old_category(session, category_id)
+ if not get_package_list_info(session, CategoryInfo.CategoryId):
+ add_old_category(session, CategoryInfo.CategoryId)
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-02-22 18:11 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-02-22 18:11 UTC (permalink / raw
To: gentoo-commits
commit: 7ee36aabda72a4f6006d0fb0810ba8be5e80d9c4
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 22 18:11:52 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb 22 18:11:52 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7ee36aab
redone remove_old_cpv job
pym/tbc/old_cpv.py | 21 +++++++--------------
1 file changed, 7 insertions(+), 14 deletions(-)
diff --git a/pym/tbc/old_cpv.py b/pym/tbc/old_cpv.py
index cc80428..3c9ffd4 100644
--- a/pym/tbc/old_cpv.py
+++ b/pym/tbc/old_cpv.py
@@ -9,8 +9,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker
from tbc.ConnectionManager import NewConnection
from tbc.readconf import read_config_settings
from tbc.sqlquerys import get_category_list_info, get_package_list_info, get_ebuild_list_info, \
- get_build_job_all, del_old_build_jobs, del_old_ebuild, add_old_package, del_old_package, \
- add_old_category
+ get_build_job_all, del_old_build_jobs, del_old_ebuild, del_old_package, add_old_category
def remove_old_ebuilds(package_id, config_id, tbc_settings, cp):
today = datetime.utcnow()
@@ -33,12 +32,13 @@ def remove_old_ebuilds(package_id, config_id, tbc_settings, cp):
del_old_build_jobs(session2, build_job.BuildJobId)
del_old_ebuild(session2, EbuildInfo.EbuildId)
if not get_ebuild_list_info(session2, package_id):
- add_old_package(session2, package_id)
+ log_msg = "Removing: %s" % (cp,)
+ write_log(session2, log_msg, "info", config_id, 'old_cpv.remove_old_cpv_ebuilds')
+ del_old_package(session2, package_id)
session2.close
Session.remove()
def remove_old_cpv_main(session, config_id):
- today = datetime.datetime.utcnow()
tbc_settings = read_config_settings()
# Use all cores when multiprocessing
#pool_cores = multiprocessing.cpu_count()
@@ -51,16 +51,9 @@ def remove_old_cpv_main(session, config_id):
PackagesInfo = get_package_list_info(session, CategoryInfo.CategoryId)
for PackageInfo in PackagesInfo:
cp = CategoryInfo.Category + '/' + PackageInfo.Package
- if PackageInfo.Active:
- # pool.apply_async( remove_old_ebuilds, (Package.PackageId, config_id, tbc_settings, cp,))
- # use this when debuging
- remove_old_ebuilds(PackageInfo.PackageId, config_id, tbc_settings, cp,)
- else:
- duration = today - PackageInfo.TimeStamp
- if duration.days > 30:
- log_msg = "Removing: %s" % (cp,)
- write_log(session2, log_msg, "info", config_id, 'old_cpv.remove_old_cpv_main')
- del_old_package(session, PackageInfo.PackageId)
+ # pool.apply_async( remove_old_ebuilds, (Package.PackageId, config_id, tbc_settings, cp,))
+ # use this when debuging
+ remove_old_ebuilds(PackageInfo.PackageId, config_id, tbc_settings, cp,)
#close and join the multiprocessing pools
# pool.close()
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-02-22 18:33 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-02-22 18:33 UTC (permalink / raw
To: gentoo-commits
commit: c5bc414fb226ee5ab38c4d0062e2a534d9b92a8a
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 22 18:34:24 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb 22 18:34:24 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=c5bc414f
remove multi cpu support in remove_old_cpv job
pym/tbc/old_cpv.py | 47 ++++++++++++++---------------------------------
pym/tbc/sqlquerys.py | 18 +++---------------
2 files changed, 17 insertions(+), 48 deletions(-)
diff --git a/pym/tbc/old_cpv.py b/pym/tbc/old_cpv.py
index 3c9ffd4..e97b121 100644
--- a/pym/tbc/old_cpv.py
+++ b/pym/tbc/old_cpv.py
@@ -2,61 +2,42 @@
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function
-import multiprocessing
from datetime import datetime
from tbc.log import write_log
-from sqlalchemy.orm import scoped_session, sessionmaker
-from tbc.ConnectionManager import NewConnection
-from tbc.readconf import read_config_settings
from tbc.sqlquerys import get_category_list_info, get_package_list_info, get_ebuild_list_info, \
get_build_job_all, del_old_build_jobs, del_old_ebuild, del_old_package, add_old_category
-def remove_old_ebuilds(package_id, config_id, tbc_settings, cp):
- today = datetime.utcnow()
- session_factory = sessionmaker(bind=NewConnection(tbc_settings))
- Session = scoped_session(session_factory)
- session2 = Session()
- EbuildsInfo = get_ebuild_list_info(session2, package_id)
+def remove_old_ebuilds(session, package_id, config_id, cp, today):
+ EbuildsInfo = get_ebuild_list_info(session, package_id)
for EbuildInfo in EbuildsInfo:
cpv = cp + '-' + EbuildInfo.Version
log_msg = "Checking: %s" % (cpv,)
- write_log(session2, log_msg, "info", config_id, 'old_cpv.remove_old_ebuilds')
+ write_log(session, log_msg, "info", config_id, 'old_cpv.remove_old_ebuilds')
if not EbuildInfo.Active:
duration = today - EbuildInfo.TimeStamp
if duration.days > 30:
log_msg = "Removing: %s" % (cpv,)
- write_log(session2, log_msg, "info", config_id, 'old_cpv.remove_old_ebuilds')
- build_job_id_list = get_build_job_all(session2, EbuildInfo.EbuildId)
+ write_log(session, log_msg, "info", config_id, 'old_cpv.remove_old_ebuilds')
+ build_job_id_list = get_build_job_all(session, EbuildInfo.EbuildId)
if build_job_id_list != []:
for build_job in build_job_id_list:
- del_old_build_jobs(session2, build_job.BuildJobId)
- del_old_ebuild(session2, EbuildInfo.EbuildId)
- if not get_ebuild_list_info(session2, package_id):
+ del_old_build_jobs(session, build_job.BuildJobId)
+ del_old_ebuild(session, EbuildInfo.EbuildId)
+ if get_ebuild_list_info(session, package_id) == []:
log_msg = "Removing: %s" % (cp,)
- write_log(session2, log_msg, "info", config_id, 'old_cpv.remove_old_cpv_ebuilds')
- del_old_package(session2, package_id)
- session2.close
- Session.remove()
+ write_log(session, log_msg, "info", config_id, 'old_cpv.remove_old_cpv_ebuilds')
+ del_old_package(session, package_id)
def remove_old_cpv_main(session, config_id):
- tbc_settings = read_config_settings()
- # Use all cores when multiprocessing
- #pool_cores = multiprocessing.cpu_count()
- #pool = multiprocessing.Pool(processes = pool_cores)
-
+ today = datetime.utcnow()
CategorysInfo = get_category_list_info(session)
for CategoryInfo in CategorysInfo:
log_msg = "Checking: %s" % (CategoryInfo.Category,)
- write_log(session2, log_msg, "info", config_id, 'old_cpv.remove_old_cpv_main')
+ write_log(session, log_msg, "info", config_id, 'old_cpv.remove_old_cpv_main')
PackagesInfo = get_package_list_info(session, CategoryInfo.CategoryId)
for PackageInfo in PackagesInfo:
cp = CategoryInfo.Category + '/' + PackageInfo.Package
- # pool.apply_async( remove_old_ebuilds, (Package.PackageId, config_id, tbc_settings, cp,))
- # use this when debuging
- remove_old_ebuilds(PackageInfo.PackageId, config_id, tbc_settings, cp,)
+ remove_old_ebuilds(session, PackageInfo.PackageId, config_id, cp, today)
- #close and join the multiprocessing pools
- # pool.close()
- # pool.join()
- if not get_package_list_info(session, CategoryInfo.CategoryId):
+ if get_package_list_info(session, CategoryInfo.CategoryId) == []:
add_old_category(session, CategoryInfo.CategoryId)
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 75e3d30..85621fe 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -585,25 +585,13 @@ def add_repoman_log(session, package_id, repoman_log, repoman_hash):
session.commit()
def get_category_list_info(session):
- try:
- CategorysInfo = session.query(Categories).all()
- except NoResultFound as e:
- return False
- return CategorysInfo
+ return session.query(Categories).all()
def get_package_list_info(session, category_id):
- try:
- PackagesInfo = session.query(Packages).filter_by(CategoryId = category_id).all()
- except NoResultFound as e:
- return False
- return PackagesInfo
+ return session.query(Packages).filter_by(CategoryId = category_id).all()
def get_ebuild_list_info(session, package_id):
- try:
- EbuildsInfo = session.query(Ebuilds).filter_by(PackageId = package_id).all()
- except NoResultFound as e:
- return False
- return EbuildsInfo
+ return session.query(Ebuilds).filter_by(PackageId = package_id).all()
def del_old_ebuild(session, ebuild_id):
session.query(EbuildsRestrictions).filter(EbuildsRestrictions.EbuildId == ebuild_id).delete()
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-02-29 23:01 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-02-29 23:01 UTC (permalink / raw
To: gentoo-commits
commit: 495e43e67bbf4a013b088e0add0b36edae644d21
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 29 23:01:59 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb 29 23:01:59 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=495e43e6
fix a bug in repoman logging multiple repeat
pym/tbc/build_log.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 8b6ef0b..d053d57 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -41,16 +41,16 @@ def check_repoman_full(session, pkgdir, package_id, config_id, cpv=False):
status = repoman_full(session, pkgdir, config_id)
repoman_hash = hashlib.sha256()
if cpv:
- element = cpv.split('/')
- pv = element[1]
+ ebuild_version_tree = portage.versions.cpv_getversion(cpv)
if status:
repoman_dict = {}
for k, v in status.items():
repoman_log2 = []
for line in v:
- if not cpv:
- repoman_log2.append(line)
- elif cpv and re.search(pv + '.ebuild', line):
+ if cpv:
+ if re.search(ebuild_version_tree, line):
+ repoman_log2.append(line)
+ else:
repoman_log2.append(line)
if not repoman_log2 == []:
repoman_dict[k] = repoman_log2
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-03-02 20:50 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-03-02 20:50 UTC (permalink / raw
To: gentoo-commits
commit: e57dd6203e4c04ac18c4b5333bf6c78b1a1eff11
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Wed Mar 2 20:50:59 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Wed Mar 2 20:50:59 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e57dd620
add more support for use test
pym/tbc/build_job.py | 49 ++++++++++++++++++++++++++++++++++++-------------
pym/tbc/db_mapping.py | 6 ++++--
pym/tbc/flags.py | 10 ++--------
pym/tbc/package.py | 18 +++++++++++++-----
pym/tbc/sqlquerys.py | 15 ++++++---------
5 files changed, 61 insertions(+), 37 deletions(-)
diff --git a/pym/tbc/build_job.py b/pym/tbc/build_job.py
index dcec3b9..e8ab829 100644
--- a/pym/tbc/build_job.py
+++ b/pym/tbc/build_job.py
@@ -67,6 +67,14 @@ class build_job_action(object):
build_cpv_list = []
depclean_fail = True
disable_test_features = False
+ enable_test_features = False
+ restrictions_test = False
+ restrictions_list= get_ebuild_restrictions(self._session, build_dict['ebuild_id'])
+ if restrictions_list:
+ if "test" in restrictions_list:
+ restrictions_test = True
+ if restrictions_test and "test" in settings.features:
+ disable_test_features = True
for k, build_use_flags_list in buildqueru_cpv_dict.items():
build_cpv_list.append("=" + k)
if not build_use_flags_list == None:
@@ -80,22 +88,37 @@ class build_job_action(object):
f.write(filetext)
f.write('\n')
f.close
- if not build_dict['build_useflags'] is None:
- if "test" in build_dict['build_useflags'] and "test" in settings.features:
- if build_dict['build_useflags']['test'] is False:
- disable_test_features = True
- restrictions_dict = get_ebuild_restrictions(self._session, build_dict['ebuild_id'])
- if restrictions_dict:
- if "test" in restrictions_dict:
- disable_test_features = True
- if disable_test_features:
- filetext = k + ' ' + 'notest.conf'
+
+ if not build_dict['build_useflags'] is None:
+ if "test" in build_dict['build_useflags']:
+ if build_dict['build_useflags']['test'] is False and "test" in settings.features:
+ disable_test_features = True
+ if build_dict['build_useflags']['test'] is True and not disable_test_features and "test" not in settings.features:
+ enable_test_features = True
+ if disable_test_features:
+ filetext = '=' + k + ' ' + 'notest.conf'
+ log_msg = "filetext: %s" % filetext
+ add_logs(self._session, log_msg, "info", self._config_id)
+ with open("/etc/portage/package.env/99_env", "a") as f:
+ f.write(filetext)
+ f.write('\n')
+ f.close
+ if enable_test_features:
+ filetext = k + ' ' + 'test.conf'
log_msg = "filetext: %s" % filetext
add_logs(self._session, log_msg, "info", self._config_id)
- with open("/etc/portage/package.env", "a") as f:
+ with open("/etc/portage/package.env/99_env", "a") as f:
f.write(filetext)
f.write('\n')
f.close
+ filetext = '=' + k + ' ' + 'test.conf'
+ log_msg = "filetext: %s" % filetext
+ add_logs(self._session, log_msg, "info", self._config_id)
+ with open("/etc/portage/package.env/99_env", "a") as f:
+ f.write(filetext)
+ f.write('\n')
+ f.close
+
log_msg = "build_cpv_list: %s" % (build_cpv_list,)
add_logs(self._session, log_msg, "info", self._config_id)
@@ -138,8 +161,8 @@ class build_job_action(object):
os.remove("/etc/portage/package.use/99_autounmask")
with open("/etc/portage/package.use/99_autounmask", "a") as f:
f.close
- os.remove("/etc/portage/package.env")
- with open("/etc/portage/package.env", "a") as f:
+ os.remove("/etc/portage/package.env/99_env")
+ with open("/etc/portage/package.env/99_env/", "a") as f:
f.close
except:
pass
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 86884bf..34ac324 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -12,11 +12,13 @@ class Keywords(Base):
KeywordId = Column('keyword_id', Integer, primary_key=True)
Keyword = Column('keyword', String)
__tablename__ = 'keywords'
-
+
class Setups(Base):
SetupId = Column('setup_id', Integer, primary_key=True)
Setup = Column('setup', String(100))
Profile = Column('profile', String(150))
+ Test = Column('test', Boolean, default=False)
+ Repoman = Column('repoman', Boolean, default=False)
__tablename__ = 'setups'
class Configs(Base):
@@ -25,7 +27,7 @@ class Configs(Base):
SetupId = Column('setup_id', Integer, ForeignKey('setups.setup_id'))
Host = Column('default_config', Boolean, default=False)
__tablename__ = 'configs'
-
+
class Logs(Base):
LogId = Column('log_id', Integer, primary_key=True)
ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id'))
diff --git a/pym/tbc/flags.py b/pym/tbc/flags.py
index 82b7d42..10c9390 100644
--- a/pym/tbc/flags.py
+++ b/pym/tbc/flags.py
@@ -1,14 +1,8 @@
-#!/usr/bin/python
-#
-# Copyright 1998-2015 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# Origin flags.py from portage public api repo
from __future__ import print_function
-from _emerge.main import parse_opts
-from _emerge.depgraph import backtrack_depgraph, depgraph, resume_depgraph
-from _emerge.create_depgraph_params import create_depgraph_params
-from _emerge.actions import load_emerge_config
import portage
import os
@@ -163,7 +157,7 @@ class tbc_use_flags(object):
iuse_flags = self.filter_flags(self.get_iuse(), use_expand_hidden, usemasked, useforced)
#flags = filter_flags(use_flags, use_expand_hidden, usemasked, useforced)
final_flags = self.filter_flags(final_use, use_expand_hidden, usemasked, useforced)
- return iuse_flags, final_flags
+ return iuse_flags, final_flags, usemasked
def get_flags_looked(self):
"""Retrieves all information needed to filter out hidden, masked, etc.
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index c069c7e..255ec67 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -56,16 +56,20 @@ class tbc_package(object):
# Get the iuse and use flags for that config/setup and cpv
init_useflags = tbc_use_flags(mysettings_setup, myportdb_setup, build_cpv)
- iuse_flags_list, final_use_list = init_useflags.get_flags()
+ iuse_flags_list, final_use_list, usemasked = init_useflags.get_flags()
iuse_flags_list2 = []
for iuse_line in iuse_flags_list:
iuse_flags_list2.append( init_useflags.reduce_flag(iuse_line))
-
+ enable_test = False
+ if SetupInfo.Test:
+ if not "test" in usemasked:
+ enable_test = True
# Dict the needed info
attDict = {}
attDict['cpv'] = build_cpv
attDict['useflags'] = final_use_list
attDict['iuse'] = iuse_flags_list2
+ attDict['test'] = enable_test
config_cpv_dict[ConfigInfo.SetupId] = attDict
# Clean some cache
@@ -156,6 +160,7 @@ class tbc_package(object):
use_flagsDict[x] = True
for x in use_disable:
use_flagsDict[x] = False
+ enable_test = v['test']
# Unpack packageDict
i = 0
for k, v in packageDict.items():
@@ -163,9 +168,12 @@ class tbc_package(object):
# Comper and add the cpv to buildqueue
if build_cpv == k:
- restrictions_dict = get_ebuild_restrictions(self._session, ebuild_id)
- if restrictions_dict:
- if "test" in restrictions_dict and "test" in use_flagsDict:
+ # check if we need to enable or disable test
+ if "test" in use_flagsDict and enable_test:
+ use_flagsDict['test'] = True
+ restrictions_list = get_ebuild_restrictions(self._session, ebuild_id)
+ if restrictions_list:
+ if "test" in restrictions_list and "test" in use_flagsDict:
use_flagsDict['test'] = False
add_new_build_job(self._session, ebuild_id, setup_id, use_flagsDict, self._config_id)
# B = Build cpv use-flags config
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 85621fe..60f4ba8 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -561,16 +561,13 @@ def get_ebuild_id_db(session, checksum, package_id, ebuild_version):
def get_ebuild_restrictions(session, ebuild_id):
restrictions = []
- try:
- EbuildsRestrictionsInfo = session.query(EbuildsRestrictions).filter_by(EbuildId = ebuild_id).one()
- except NoResultFound as e:
+ EbuildsRestrictionsInfos = session.query(EbuildsRestrictions).filter_by(EbuildId = ebuild_id).all()
+ if EbuildsRestrictionsInfos == []:
return False
- except MultipleResultsFound as e:
- EbuildsRestrictionsInfos = session.query(EbuildsRestrictions).filter_by(EbuildId = ebuild_id).all()
- for EbuildsRestrictionsInfo in EbuildsRestrictionsInfos:
- restrictions.append(session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one())
- return restrictions
- return restrictions.append(session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one())
+ for EbuildsRestrictionsInfo in EbuildsRestrictionsInfos:
+ RestrictionsInfo = session.query(Restrictions).filter_by(RestrictionId = EbuildsRestrictionsInfo.RestrictionId).one()
+ restrictions.append(RestrictionsInfo.Restriction)
+ return restrictions
def add_repoman_log(session, package_id, repoman_log, repoman_hash):
try:
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-03-03 15:26 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-03-03 15:26 UTC (permalink / raw
To: gentoo-commits
commit: 75fd66c5a04ba87ae92005b335fd5f854e00b5e5
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Mar 3 15:27:15 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Mar 3 15:27:15 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=75fd66c5
fix test too many values to unpack
pym/tbc/flags.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/tbc/flags.py b/pym/tbc/flags.py
index 10c9390..eb3f782 100644
--- a/pym/tbc/flags.py
+++ b/pym/tbc/flags.py
@@ -195,7 +195,7 @@ class tbc_use_flags(object):
return iuse_flags, final_flags
def comper_useflags(self, build_dict):
- iuse_flags, use_enable = self.get_flags()
+ iuse_flags, use_enable, usemasked = self.get_flags()
iuse = []
build_use_flags_dict = build_dict['build_useflags']
build_use_flags_list = []
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-03-06 17:47 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-03-06 17:47 UTC (permalink / raw
To: gentoo-commits
commit: 2fe8d223790b5f40b42e41f3ed80b59d408aaa05
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 6 17:47:58 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Mar 6 17:47:58 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2fe8d223
fix a typo for package env files
pym/tbc/build_job.py | 23 ++++++++---------------
1 file changed, 8 insertions(+), 15 deletions(-)
diff --git a/pym/tbc/build_job.py b/pym/tbc/build_job.py
index e8ab829..d0d72c3 100644
--- a/pym/tbc/build_job.py
+++ b/pym/tbc/build_job.py
@@ -103,21 +103,14 @@ class build_job_action(object):
f.write(filetext)
f.write('\n')
f.close
- if enable_test_features:
- filetext = k + ' ' + 'test.conf'
- log_msg = "filetext: %s" % filetext
- add_logs(self._session, log_msg, "info", self._config_id)
- with open("/etc/portage/package.env/99_env", "a") as f:
- f.write(filetext)
- f.write('\n')
- f.close
- filetext = '=' + k + ' ' + 'test.conf'
- log_msg = "filetext: %s" % filetext
- add_logs(self._session, log_msg, "info", self._config_id)
- with open("/etc/portage/package.env/99_env", "a") as f:
- f.write(filetext)
- f.write('\n')
- f.close
+ if enable_test_features:
+ filetext = '=' + k + ' ' + 'test.conf'
+ log_msg = "filetext: %s" % filetext
+ add_logs(self._session, log_msg, "info", self._config_id)
+ with open("/etc/portage/package.env/99_env", "a") as f:
+ f.write(filetext)
+ f.write('\n')
+ f.close
log_msg = "build_cpv_list: %s" % (build_cpv_list,)
add_logs(self._session, log_msg, "info", self._config_id)
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-11-15 21:58 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-11-15 21:58 UTC (permalink / raw
To: gentoo-commits
commit: 7af6ebcf89684464488684ee3dc349827ddd7f46
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Apr 5 21:35:47 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Apr 5 21:35:47 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7af6ebcf
remove log file from summery text
pym/tbc/build_log.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 5476592..2a00b95 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -292,16 +292,17 @@ def add_buildlog_main(settings, pkg, trees):
build_log_dict = {}
build_log_dict = get_buildlog_info(session, settings, pkg, build_dict, config_id)
error_log_list = build_log_dict['error_log_list']
+ build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1]
build_error = ""
log_hash = hashlib.sha256()
build_error = ""
if error_log_list != []:
for log_line in error_log_list:
- build_error = build_error + log_line
+ if not re.search(build_log_dict['logfilename'], log_line):
+ build_error = build_error + log_line
log_hash.update(build_error.encode('utf-8'))
build_log_dict['build_error'] = build_error
build_log_dict['log_hash'] = log_hash.hexdigest()
- build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1]
log_msg = "Logfile name: %s" % (settings.get("PORTAGE_LOG_FILE"),)
write_log(session, log_msg, "info", config_id, 'build_log.add_buildlog_main')
build_log_dict['emerge_info'] = get_emerge_info_id(settings, trees, session, config_id)
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-11-20 15:37 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-11-20 15:37 UTC (permalink / raw
To: gentoo-commits
commit: cc61c64c0c2bfb3bdfe32eb0030db2ab69483346
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Nov 20 15:37:42 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Nov 20 15:37:42 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=cc61c64c
ImportError: No module named 'repoman.checks.ebuilds'
pym/tbc/qachecks.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/tbc/qachecks.py b/pym/tbc/qachecks.py
index 892642f..976cb2a 100644
--- a/pym/tbc/qachecks.py
+++ b/pym/tbc/qachecks.py
@@ -14,7 +14,7 @@ from portage import os, _encodings, _unicode_decode, _unicode_encode
from portage.exception import DigestException, FileNotFound, ParseError, PermissionDenied
from _emerge.Package import Package
from _emerge.RootConfig import RootConfig
-from repoman.checks.ebuilds.checks import run_checks
+from repoman.modules.scan.ebuild.checks import run_checks
from tbc.repoman import repoman_main
from tbc.sqlquerys import get_configmetadata_info, get_config_info, get_setup_info
import portage
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-05 21:11 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-05 21:11 UTC (permalink / raw
To: gentoo-commits
commit: d88b81146838c007345ca0c87cd6d51b6f624f27
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Dec 5 21:11:08 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Dec 5 21:11:08 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=d88b8114
Set email to maintainer-needed if email not found
pym/tbc/package.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 255ec67..6e42af3 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -211,7 +211,9 @@ class tbc_package(object):
if md_email_list != []:
attDict['metadata_xml_email'] = md_email_list
else:
- log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
+ md_email_list.append('maintainer-needed@gentoo.org')
+ attDict['metadata_xml_email'] = md_email_list
+ log_msg = "Metadata file %s missing Email. Setting it to maintainer-needed" % (pkgdir + "/metadata.xml")
write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_package_metadataDict')
attDict['git_changlog'] = self.get_git_changelog_text(repodir, cp)
attDict['metadata_xml_descriptions'] = ''
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-10 16:17 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-10 16:17 UTC (permalink / raw
To: gentoo-commits
commit: a18f23061cd8ca410e6453212bf802feb8d0c602
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat Dec 10 16:16:56 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat Dec 10 16:16:56 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=a18f2306
add port for irker
pym/tbc/irk.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/tbc/irk.py b/pym/tbc/irk.py
index 3186cb5..b00f816 100644
--- a/pym/tbc/irk.py
+++ b/pym/tbc/irk.py
@@ -23,6 +23,6 @@ def irk(target, message, server = DEFAULT_SERVER):
def send_irk(msg, host):
target = "tinderbox-cluster"
try:
- irk(target, msg, server = host)
+ irk(target, msg, server = (host, 6659)
except socket.error as e:
sys.stderr.write("irk: write to server failed: %r\n" % e)
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-20 20:25 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-20 20:25 UTC (permalink / raw
To: gentoo-commits
commit: f028abe51998fb054cf463861c920babed43a4cf
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 19:57:16 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 19:57:16 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f028abe5
fix a typo in irk code
pym/tbc/irk.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/tbc/irk.py b/pym/tbc/irk.py
index b00f816..3b3589b 100644
--- a/pym/tbc/irk.py
+++ b/pym/tbc/irk.py
@@ -23,6 +23,6 @@ def irk(target, message, server = DEFAULT_SERVER):
def send_irk(msg, host):
target = "tinderbox-cluster"
try:
- irk(target, msg, server = (host, 6659)
+ irk(target, msg, server = (host, 6659))
except socket.error as e:
sys.stderr.write("irk: write to server failed: %r\n" % e)
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-20 20:25 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-20 20:25 UTC (permalink / raw
To: gentoo-commits
commit: 7df70c0bf6d0a8cd4a2773091778a44ada419294
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 19:59:12 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 19:59:12 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7df70c0b
add new,update on ebuilds and package
pym/tbc/db_mapping.py | 2 ++
pym/tbc/package.py | 8 +++++---
pym/tbc/sqlquerys.py | 4 ++--
3 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index d3fead1..304dcd8 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -106,6 +106,7 @@ class PackagesMetadata(Base):
PackageId = Column('package_id', Integer, ForeignKey('packages.package_id'))
Gitlog = Column('gitlog', Text)
Descriptions = Column('descriptions', Text)
+ New = Column('new', Boolean, default=False)
__tablename__ = 'packages_metadata'
class Ebuilds(Base):
@@ -287,6 +288,7 @@ class EbuildsMetadata(Base):
EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
Commit = Column('commit', String(100))
New = Column('new', Boolean, default=False)
+ Updated = Column('updated', Boolean, default=False)
Descriptions = Column('descriptions', Text)
__tablename__ = 'ebuilds_metadata'
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 6e42af3..0a5749d 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -141,6 +141,7 @@ class tbc_package(object):
#attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
attDict['git_commit'] = git_commit
attDict['new'] = False
+ attDict['update'] = False
attDict['ebuild_version_descriptions_tree'] = ebuild_version_metadata_tree[7]
return attDict
@@ -195,7 +196,6 @@ class tbc_package(object):
package_metadataDict = {}
md_email_list = []
herd = None
- attDict['metadata_xml_email'] = False
try:
pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
except:
@@ -217,6 +217,7 @@ class tbc_package(object):
write_log(self._session, log_msg, "warning", self._config_id, 'packages.get_package_metadataDict')
attDict['git_changlog'] = self.get_git_changelog_text(repodir, cp)
attDict['metadata_xml_descriptions'] = ''
+ attDict['new'] = False
package_metadataDict[package_id] = attDict
return package_metadataDict
@@ -288,12 +289,12 @@ class tbc_package(object):
old_ebuild_id_list = []
for cpv in sorted(ebuild_list_tree):
packageDict[cpv] = self.get_packageDict(pkgdir, cpv, repo)
+ packageDict[cpv]['new'] = True
# take package descriptions from the ebuilds
if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
-
- packageDict[cpv]['new'] = True
+ package_metadataDict[package_id][new] = True
self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list)
log_msg = "C %s:%s ... Done." % (cp, repo)
write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_package_db')
@@ -382,6 +383,7 @@ class tbc_package(object):
# U = Updated ebuild
log_msg = "U %s:%s" % (cpv, repo,)
write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
+ packageDict[cpv]['updated'] = True
package_updated = True
else:
# Remove cpv from packageDict and add ebuild to new ebuils list
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 60f4ba8..770e84e 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -423,7 +423,7 @@ def add_new_ebuild_sql(session, packageDict):
except (MultipleResultsFound) as e:
# FIXME
sys.exit()
- session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Commit = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
+ session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
session.commit()
ebuild_id_list.append(EbuildInfo.EbuildId)
restrictions = []
@@ -505,7 +505,7 @@ def update_package_metadata(session, package_metadataDict):
try:
PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = k).one()
except NoResultFound as e:
- session.add(PackagesMetadata(PackageId = k, Gitlog = v['git_changlog'], Descriptions = v['metadata_xml_descriptions']))
+ session.add(PackagesMetadata(PackageId = k, Gitlog = v['git_changlog'], Descriptions = v['metadata_xml_descriptions'], v['new']))
session.commit()
else:
PackagesMetadataInfo.Gitlog = v['git_changlog']
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-20 20:25 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-20 20:25 UTC (permalink / raw
To: gentoo-commits
commit: ccd8dad636b0f58585c0c3c5eac7f80d138600ee
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 20:25:03 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 20:25:03 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=ccd8dad6
add Waiting_on_guest
pym/tbc/db_mapping.py | 2 +-
pym/tbc/jobs.py | 4 ++--
pym/tbc/sqlquerys.py | 6 ++++++
pym/tbc/sync.py | 5 ++++-
pym/tbc/updatedb.py | 4 +++-
5 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 304dcd8..aba6ee1 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -39,7 +39,7 @@ class Logs(Base):
class Jobs(Base):
JobId = Column('job_id', Integer, primary_key=True)
JobType = Column('job_type', Enum('updatedb', 'esync', 'removeold_cpv'))
- Status = Column('status', Enum('Runing', 'Done', 'Waiting'))
+ Status = Column('status', Enum('Runing', 'Done', 'Waiting', 'Waiting_on_guest'))
User = Column('user', String(50))
ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id'))
RunConfigId = Column('run_config_id', Integer, ForeignKey('configs.config_id'))
diff --git a/pym/tbc/jobs.py b/pym/tbc/jobs.py
index efe8623..800fe87 100644
--- a/pym/tbc/jobs.py
+++ b/pym/tbc/jobs.py
@@ -46,7 +46,7 @@ def jobs_main(session, config_id):
# log_msg = "Job %s did fail." % (job_id,)
# write_log(session, log_msg, "info", config_id, 'jobs_main')
elif job == "esync":
- update_job_list(session, "Runing", job_id)
+ update_job_list(session, "Waiting_on_guest", job_id)
log_msg = "Job %s is runing." % (job_id,)
write_log(session, log_msg, "info", config_id, 'jobs_main')
if update_db_main(session, git_sync_main(session), config_id):
@@ -58,7 +58,7 @@ def jobs_main(session, config_id):
log_msg = "Job %s did fail." % (job_id,)
write_log(session, log_msg, "info", config_id, 'jobs_main')
elif job == "updatedb":
- update_job_list(session, "Runing", job_id)
+ update_job_list(session, "Waiting_on_guest", job_id)
log_msg = "Job %s is runing." % (job_id,)
write_log(session, log_msg, "info", config_id, 'jobs_main')
if update_db_main(session, None, config_id):
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 770e84e..901db08 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -44,6 +44,12 @@ def get_jobs(session, config_id):
return None
return JobsInfo
+def get_job_status_waiting_on_guest(session):
+ JobsInfo = session.query(Jobs).filter_by(Status = 'Waiting_on_guest').one()
+ if JobsInfo == []:
+ return None
+ return JobInfo.JobId
+
def update_job_list(session, status, job_id):
JobInfo = session.query(Jobs).filter_by(JobId = job_id).one()
JobInfo.Status = status
diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 7063bfb..dee13ff 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -11,7 +11,8 @@ import re
import git
from tbc.sqlquerys import get_config_id_fqdn, add_logs, get_config_all_info, \
- get_configmetadata_info, get_config_info, get_setup_info
+ get_configmetadata_info, get_config_info, get_setup_info, get_job_status_waiting_on_guest, \
+ update_job_list
from tbc.readconf import read_config_settings
from tbc.log import write_log
@@ -64,6 +65,8 @@ def git_sync_main(session):
else:
time.sleep(60)
+ job_id = get_job_status_waiting_on_guest(session)
+ update_job_list(session, 'Runing', job_id)
# check git diffs witch get updated and pass that to a dict
# fetch and merge the repo
repo_cp_dict = {}
diff --git a/pym/tbc/updatedb.py b/pym/tbc/updatedb.py
index 0753d92..b6539e4 100644
--- a/pym/tbc/updatedb.py
+++ b/pym/tbc/updatedb.py
@@ -13,7 +13,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker
from tbc.ConnectionManager import NewConnection
from tbc.sqlquerys import get_package_info, update_repo_db, \
update_categories_db, get_configmetadata_info, get_config_all_info, add_new_build_job, \
- get_config_info, get_setup_info
+ get_config_info, get_setup_info, get_job_status_waiting_on_guest, update_job_list
from tbc.check_setup import check_make_conf
from tbc.package import tbc_package
# Get the options from the config file tbc.conf
@@ -93,6 +93,8 @@ def update_cpv_db(session, repo_cp_dict, config_id, tbc_settings):
break
time.sleep(30)
+ job_id = get_job_status_waiting_on_guest(session)
+ update_job_list(session, 'Runing', job_id)
log_msg = "Checking categories, package, ebuilds"
write_log(session, log_msg, "info", config_id, 'updatedb.update_cpv_db')
new_build_jobs_list = []
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-20 21:16 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-20 21:16 UTC (permalink / raw
To: gentoo-commits
commit: 603f968772a11e3406f774967f76bbbcb5ece729
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 21:15:24 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 21:15:24 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=603f9687
fix a typo in add_new_ebuild_sql
pym/tbc/sqlquerys.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 901db08..aa9c2a7 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -429,7 +429,7 @@ def add_new_ebuild_sql(session, packageDict):
except (MultipleResultsFound) as e:
# FIXME
sys.exit()
- session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
+ session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], Commit = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
session.commit()
ebuild_id_list.append(EbuildInfo.EbuildId)
restrictions = []
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-20 21:21 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-20 21:21 UTC (permalink / raw
To: gentoo-commits
commit: 7c8cea8c8af2b8393b59ac1bf23917c130f9b369
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 21:20:30 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 21:20:30 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7c8cea8c
fix a typo in update_package_metadata
pym/tbc/sqlquerys.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index aa9c2a7..0276131 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -511,7 +511,7 @@ def update_package_metadata(session, package_metadataDict):
try:
PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = k).one()
except NoResultFound as e:
- session.add(PackagesMetadata(PackageId = k, Gitlog = v['git_changlog'], Descriptions = v['metadata_xml_descriptions'], v['new']))
+ session.add(PackagesMetadata(PackageId = k, Gitlog = v['git_changlog'], Descriptions = v['metadata_xml_descriptions'], New = v['new']))
session.commit()
else:
PackagesMetadataInfo.Gitlog = v['git_changlog']
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-20 22:13 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-20 22:13 UTC (permalink / raw
To: gentoo-commits
commit: 966f66269f41f46f29895f7c9b6240f7babd5094
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Dec 20 22:13:17 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Dec 20 22:13:17 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=966f6626
fix a NoResultFound in get_job_status_waiting_on_guest
pym/tbc/sqlquerys.py | 5 +++--
pym/tbc/sync.py | 4 +++-
pym/tbc/updatedb.py | 4 +++-
3 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 0276131..0ec5fc9 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -45,8 +45,9 @@ def get_jobs(session, config_id):
return JobsInfo
def get_job_status_waiting_on_guest(session):
- JobsInfo = session.query(Jobs).filter_by(Status = 'Waiting_on_guest').one()
- if JobsInfo == []:
+ try:
+ JobsInfo = session.query(Jobs).filter_by(Status = 'Waiting_on_guest').one()
+ except NoResultFound as e:
return None
return JobInfo.JobId
diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index dee13ff..a93b84f 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -66,7 +66,9 @@ def git_sync_main(session):
time.sleep(60)
job_id = get_job_status_waiting_on_guest(session)
- update_job_list(session, 'Runing', job_id)
+ if not job_id is None:
+ update_job_list(session, 'Runing', job_id)
+
# check git diffs witch get updated and pass that to a dict
# fetch and merge the repo
repo_cp_dict = {}
diff --git a/pym/tbc/updatedb.py b/pym/tbc/updatedb.py
index b6539e4..7fce867 100644
--- a/pym/tbc/updatedb.py
+++ b/pym/tbc/updatedb.py
@@ -94,7 +94,9 @@ def update_cpv_db(session, repo_cp_dict, config_id, tbc_settings):
time.sleep(30)
job_id = get_job_status_waiting_on_guest(session)
- update_job_list(session, 'Runing', job_id)
+ if not job_id is None:
+ update_job_list(session, 'Runing', job_id)
+
log_msg = "Checking categories, package, ebuilds"
write_log(session, log_msg, "info", config_id, 'updatedb.update_cpv_db')
new_build_jobs_list = []
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-22 10:11 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-22 10:11 UTC (permalink / raw
To: gentoo-commits
commit: 682b192ef7f23ddebb56206e263a540e557ba515
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 09:58:30 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 09:58:30 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=682b192e
change new to updated in the update run and some typos
pym/tbc/package.py | 5 ++---
pym/tbc/sqlquerys.py | 2 +-
2 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 0a5749d..3c5664c 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -141,7 +141,7 @@ class tbc_package(object):
#attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
attDict['git_commit'] = git_commit
attDict['new'] = False
- attDict['update'] = False
+ attDict['updated'] = False
attDict['ebuild_version_descriptions_tree'] = ebuild_version_metadata_tree[7]
return attDict
@@ -264,7 +264,6 @@ class tbc_package(object):
mytree = []
mytree.append(repodir)
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
-
package_id = add_new_package_sql(self._session, cp, repo)
# Check cp with repoman full
@@ -377,7 +376,7 @@ class tbc_package(object):
# N = New ebuild
log_msg = "N %s:%s" % (cpv, repo,)
write_log(self._session, log_msg, "info", self._config_id, 'packages.update_package_db')
- packageDict[cpv]['new'] = True
+ packageDict[cpv]['updated'] = True
package_updated = True
elif ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
# U = Updated ebuild
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 0ec5fc9..c9141ef 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -49,7 +49,7 @@ def get_job_status_waiting_on_guest(session):
JobsInfo = session.query(Jobs).filter_by(Status = 'Waiting_on_guest').one()
except NoResultFound as e:
return None
- return JobInfo.JobId
+ return JobsInfo.JobId
def update_job_list(session, status, job_id):
JobInfo = session.query(Jobs).filter_by(JobId = job_id).one()
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-22 10:11 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-22 10:11 UTC (permalink / raw
To: gentoo-commits
commit: 4834c9751340622feb34f4c3d747517826499788
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 10:10:27 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 10:10:27 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=4834c975
add git commit msg
pym/tbc/db_mapping.py | 1 +
pym/tbc/package.py | 14 ++++++++++----
pym/tbc/sqlquerys.py | 2 +-
3 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index aba6ee1..964df54 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -287,6 +287,7 @@ class EbuildsMetadata(Base):
Id = Column('id', Integer, primary_key=True)
EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
Commit = Column('commit', String(100))
+ CommitMsg = Column('git_commit_msg', String(200))
New = Column('new', Boolean, default=False)
Updated = Column('updated', Boolean, default=False)
Descriptions = Column('descriptions', Text)
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 3c5664c..4bbb63e 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -92,10 +92,14 @@ class tbc_package(object):
def get_git_log_ebuild(self, repodir, ebuild_file):
git_log_ebuild = ''
g = git.Git(repodir)
+ index = 1
+ git_log_dict = {}
for line in g.log('-n 1', ebuild_file).splitlines():
- if re.search('^commit', line):
- git_log_ebuild = re.sub('commit', '', line)
- return git_log_ebuild
+ git_log_dict[index] = line
+ index = index + 1
+ git_ebuild_commit = re.sub('commit ', '', git_log_dict[1])
+ git_ebuild_commit_msg = re.sub(' ', '', git_log_dict[5])
+ return git_ebuild_commit, git_ebuild_commit_msg
def get_packageDict(self, pkgdir, cpv, repo):
@@ -115,9 +119,10 @@ class tbc_package(object):
log_msg = "C %s:%s ... Fail." % (cpv, repo)
write_log(self._session, log_msg, "info", self._config_id, 'packages.get_packageDict')
git_commit = '0'
+ git_commit_msg = '0'
else:
repodir =self._myportdb.getRepositoryPath(repo)
- git_commit = self.get_git_log_ebuild(repodir, ebuild_file)
+ git_commit, git_commit_msg = self.get_git_log_ebuild(repodir, ebuild_file)
# Get the ebuild metadata
ebuild_version_metadata_tree = self.get_ebuild_metadata(cpv, repo)
@@ -140,6 +145,7 @@ class tbc_package(object):
attDict['ebuild_version_metadata_tree'] = ebuild_version_metadata_tree
#attDict['ebuild_version_text_tree'] = ebuild_version_text_tree[0]
attDict['git_commit'] = git_commit
+ attDict['git_commit_msg'] = git_commit_msg
attDict['new'] = False
attDict['updated'] = False
attDict['ebuild_version_descriptions_tree'] = ebuild_version_metadata_tree[7]
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index c9141ef..1d69151 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -430,7 +430,7 @@ def add_new_ebuild_sql(session, packageDict):
except (MultipleResultsFound) as e:
# FIXME
sys.exit()
- session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], Commit = v['git_commit'], Descriptions = v['ebuild_version_descriptions_tree']))
+ session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], Commit = v['git_commit'], CommitMsg = v['git_commit_msg'], Descriptions = v['ebuild_version_descriptions_tree']))
session.commit()
ebuild_id_list.append(EbuildInfo.EbuildId)
restrictions = []
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-22 10:15 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-22 10:15 UTC (permalink / raw
To: gentoo-commits
commit: f8bbed36aa7ed37a73b7d06347a9b8558c8dc728
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 10:14:34 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 10:14:34 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f8bbed36
fix commit to git_commit
pym/tbc/db_mapping.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 964df54..91ae5dd 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -286,7 +286,7 @@ class EbuildsKeywords(Base):
class EbuildsMetadata(Base):
Id = Column('id', Integer, primary_key=True)
EbuildId = Column('ebuild_id', ForeignKey('ebuilds.ebuild_id'))
- Commit = Column('commit', String(100))
+ Commit = Column('git_commit', String(100))
CommitMsg = Column('git_commit_msg', String(200))
New = Column('new', Boolean, default=False)
Updated = Column('updated', Boolean, default=False)
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-22 11:29 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-22 11:29 UTC (permalink / raw
To: gentoo-commits
commit: 3ac0465df0049240f20ccb0efee65463203f7542
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 11:09:50 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 11:09:50 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=3ac0465d
reset new and updated on ebuild and package metadata
pym/tbc/updatedb.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/pym/tbc/updatedb.py b/pym/tbc/updatedb.py
index 7fce867..437cfd5 100644
--- a/pym/tbc/updatedb.py
+++ b/pym/tbc/updatedb.py
@@ -13,7 +13,8 @@ from sqlalchemy.orm import scoped_session, sessionmaker
from tbc.ConnectionManager import NewConnection
from tbc.sqlquerys import get_package_info, update_repo_db, \
update_categories_db, get_configmetadata_info, get_config_all_info, add_new_build_job, \
- get_config_info, get_setup_info, get_job_status_waiting_on_guest, update_job_list
+ get_config_info, get_setup_info, get_job_status_waiting_on_guest, update_job_list, \
+ reset_new_updated
from tbc.check_setup import check_make_conf
from tbc.package import tbc_package
# Get the options from the config file tbc.conf
@@ -156,6 +157,7 @@ def update_cpv_db(session, repo_cp_dict, config_id, tbc_settings):
def update_db_main(session, repo_cp_dict, config_id):
# Main
+ reset_new_updated(session)
if repo_cp_dict == {}:
return True
# Logging
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-22 11:29 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-22 11:29 UTC (permalink / raw
To: gentoo-commits
commit: 1513ff38cfc184098d7a96e22a5ecaec72f863d1
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 11:23:44 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 11:23:44 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=1513ff38
update timestamp when jobs is done
pym/tbc/sqlquerys.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index a64c5ef..6542c0a 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -54,6 +54,8 @@ def get_job_status_waiting_on_guest(session):
def update_job_list(session, status, job_id):
JobInfo = session.query(Jobs).filter_by(JobId = job_id).one()
JobInfo.Status = status
+ if status = 'Done':
+ JobInfo.TimeStamp = datetime.datetime.utcnow()
session.commit()
def get_config_all_info(session):
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-22 11:29 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-22 11:29 UTC (permalink / raw
To: gentoo-commits
commit: e8fbf308450b9b257f1cd16532591caba51cc192
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 11:09:24 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 11:09:24 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=e8fbf308
reset new and updated on ebuild and package metadata
pym/tbc/sqlquerys.py | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 1d69151..a64c5ef 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -616,3 +616,30 @@ def add_old_category(session, category_id):
CategorysInfo = session.query(Categories).filter_by(CategoryId = category_id).one()
CategorysInfo.Active = False
session.commit()
+
+def reset_new_updated(session):
+ try:
+ PMInfo = session.query(PackagesMetadata).filter(PackagesMetadata.New == True).all()
+ except NoResultFound as e:
+ pass
+ else:
+ for x in PMInfo:
+ x.New = False
+ session.flush()
+ try:
+ EMInfo = session.query(EbuildsMetadata).filter(EbuildsMetadata.New == True).all()
+ except NoResultFound as e:
+ pass
+ else:
+ for x in EMInfo:
+ x.New = False
+ session.flush()
+ try:
+ EMInfo = session.query(EbuildsMetadata).filter(EbuildsMetadata.Updated == True).all()
+ except NoResultFound as e:
+ pass
+ else:
+ for x in EMInfo:
+ x.Updated = False
+ session.flush()
+ session.commit()
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2016-12-22 18:31 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2016-12-22 18:31 UTC (permalink / raw
To: gentoo-commits
commit: 35875a290ff4f094f6a9f2299f2c9e9c301482d9
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Thu Dec 22 18:31:02 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Thu Dec 22 18:31:02 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=35875a29
add new on BuildLogs and BuildJobs
pym/tbc/db_mapping.py | 2 ++
pym/tbc/sqlquerys.py | 22 +++++++++++++++++++---
2 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 91ae5dd..96ae75b 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -136,6 +136,7 @@ class BuildJobs(Base):
Status = Column('status', Enum('Waiting','Building','Looked',))
BuildNow = Column('build_now', Boolean, default=False)
RemoveBin = Column('removebin', Boolean ,default=False)
+ New = Column('new', Boolean, default=False)
TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
__tablename__ = 'build_jobs'
@@ -188,6 +189,7 @@ class BuildLogs(Base):
SummeryText = Column('summery_text', Text)
LogHash = Column('log_hash', String(100))
BugId = Column('bug_id', Integer, default=0)
+ New = Column('new', Boolean, default=False)
TimeStamp = Column('time_stamp', DateTime, nullable=False, default=datetime.datetime.utcnow)
__tablename__ = 'build_logs'
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index 6542c0a..de60af7 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -54,7 +54,7 @@ def get_job_status_waiting_on_guest(session):
def update_job_list(session, status, job_id):
JobInfo = session.query(Jobs).filter_by(JobId = job_id).one()
JobInfo.Status = status
- if status = 'Done':
+ if status == 'Done':
JobInfo.TimeStamp = datetime.datetime.utcnow()
session.commit()
@@ -244,7 +244,7 @@ def add_new_buildlog(session, build_dict, build_log_dict):
return None, False
def build_log_id_no_match(build_dict, build_log_dict):
- NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = build_log_dict['fail'], SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash'])
+ NewBuildLog = BuildLogs(EbuildId = build_dict['ebuild_id'], Fail = build_log_dict['fail'], SummeryText = build_log_dict['build_error'], LogHash = build_log_dict['log_hash'], New = True)
session.add(NewBuildLog)
session.flush()
build_log_id = NewBuildLog.BuildLogId
@@ -532,7 +532,7 @@ def get_package_info_from_package_id(session, package_id):
return PackageInfo, CategoryInfo, RepoInfo
def add_new_build_job(session, ebuild_id, setup_id, use_flagsDict, config_id):
- NewBuildJobs =BuildJobs(EbuildId = ebuild_id, SetupId = setup_id, ConfigId = config_id, Status = 'Waiting', BuildNow = False, RemoveBin = True)
+ NewBuildJobs = BuildJobs(EbuildId = ebuild_id, SetupId = setup_id, ConfigId = config_id, Status = 'Waiting', BuildNow = False, RemoveBin = True, New = True)
session.add(NewBuildJobs)
session.flush()
build_job_id = NewBuildJobs.BuildJobId
@@ -636,6 +636,22 @@ def reset_new_updated(session):
for x in EMInfo:
x.New = False
session.flush()
+try:
+ BLInfo = session.query(BuildLogs).filter(BuildLogs.New == True).all()
+ except NoResultFound as e:
+ pass
+ else:
+ for x in BLInfo:
+ x.New = False
+ session.flush()
+ try:
+ BJInfo = session.query(BuildJobs).filter(BuildJobs.New == True).all()
+ except NoResultFound as e:
+ pass
+ else:
+ for x in BJInfo:
+ x.New = False
+ session.flush()
try:
EMInfo = session.query(EbuildsMetadata).filter(EbuildsMetadata.Updated == True).all()
except NoResultFound as e:
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2017-01-08 15:33 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2017-01-08 15:33 UTC (permalink / raw
To: gentoo-commits
commit: 9ea28d6fc2bc00eaf64739cf6739ff52bbfe1056
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 8 15:32:49 2017 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 8 15:32:49 2017 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=9ea28d6f
add header.txt to ignore files
pym/tbc/sync.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index a93b84f..8488c50 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -72,7 +72,7 @@ def git_sync_main(session):
# check git diffs witch get updated and pass that to a dict
# fetch and merge the repo
repo_cp_dict = {}
- search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts', '^skel.', ]
+ search_list = [ '^metadata', '^eclass', '^licenses', '^profiles', '^scripts', '^skel.', '^header.txt']
for repo_dir in git_repos_list(myportdb):
reponame = myportdb.getRepositoryName(repo_dir)
repo = git.Repo(repo_dir)
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2017-01-08 15:33 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2017-01-08 15:33 UTC (permalink / raw
To: gentoo-commits
commit: 7d547c4e12884b4f2914eb9e151d3f043978ba42
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 8 15:31:57 2017 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 8 15:31:57 2017 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7d547c4e
add slot and homepage
pym/tbc/db_mapping.py | 3 +++
pym/tbc/package.py | 2 +-
pym/tbc/sqlquerys.py | 6 ++++--
3 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/pym/tbc/db_mapping.py b/pym/tbc/db_mapping.py
index 96ae75b..f48ef56 100644
--- a/pym/tbc/db_mapping.py
+++ b/pym/tbc/db_mapping.py
@@ -293,6 +293,9 @@ class EbuildsMetadata(Base):
New = Column('new', Boolean, default=False)
Updated = Column('updated', Boolean, default=False)
Descriptions = Column('descriptions', Text)
+ Slot = Column('slot', String(10))
+ Homepage = Column('homepage', String(200))
+ License = Column('license', String(200))
__tablename__ = 'ebuilds_metadata'
class TbcConfig(Base):
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 4bbb63e..721f9b1 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -299,7 +299,7 @@ class tbc_package(object):
# take package descriptions from the ebuilds
if package_metadataDict[package_id]['metadata_xml_descriptions'] != packageDict[cpv]['ebuild_version_descriptions_tree']:
package_metadataDict[package_id]['metadata_xml_descriptions'] = packageDict[cpv]['ebuild_version_descriptions_tree']
- package_metadataDict[package_id][new] = True
+ package_metadataDict[package_id]['new'] = True
self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list)
log_msg = "C %s:%s ... Done." % (cp, repo)
write_log(self._session, log_msg, "info", self._config_id, 'packages.add_new_package_db')
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index de60af7..9c962f4 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -432,7 +432,9 @@ def add_new_ebuild_sql(session, packageDict):
except (MultipleResultsFound) as e:
# FIXME
sys.exit()
- session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], Commit = v['git_commit'], CommitMsg = v['git_commit_msg'], Descriptions = v['ebuild_version_descriptions_tree']))
+ session.add(EbuildsMetadata(EbuildId = EbuildInfo.EbuildId, New = v['new'], Updated = v['updated'], Commit = v['git_commit'], \
+ CommitMsg = v['git_commit_msg'], Descriptions = v['ebuild_version_descriptions_tree'], Slot = v['ebuild_version_metadata_tree'][2], \
+ Homepage = v['ebuild_version_metadata_tree'][5], License = v['ebuild_version_metadata_tree'][6]))
session.commit()
ebuild_id_list.append(EbuildInfo.EbuildId)
restrictions = []
@@ -636,7 +638,7 @@ def reset_new_updated(session):
for x in EMInfo:
x.New = False
session.flush()
-try:
+ try:
BLInfo = session.query(BuildLogs).filter(BuildLogs.New == True).all()
except NoResultFound as e:
pass
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2017-01-15 19:57 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2017-01-15 19:57 UTC (permalink / raw
To: gentoo-commits
commit: 700a263237acc4beb17442ff66b4e70dd0947bce
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 15 19:57:10 2017 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 15 19:57:10 2017 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=700a2632
update web irker adress
pym/tbc/build_log.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 2a00b95..762ebd4 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -328,7 +328,7 @@ def add_buildlog_main(settings, pkg, trees):
else:
build_msg = "BUILD: FAILD"
tbc_config = get_tbc_config(session)
- msg = "Package: %s Repo: %s %s %s %s Weblink http://%s/new/logs/build/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, tbc_config.WebIrker, log_id,)
+ msg = "Package: %s Repo: %s %s %s %s Weblink http://%s/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, tbc_config.WebIrker, log_id,)
write_log(session, msg, "info", config_id, 'build_log.add_buildlog_main')
send_irk(msg, tbc_config.HostIrker)
session.close
^ permalink raw reply related [flat|nested] 45+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/
@ 2018-02-05 2:52 Magnus Granberg
0 siblings, 0 replies; 45+ messages in thread
From: Magnus Granberg @ 2018-02-05 2:52 UTC (permalink / raw
To: gentoo-commits
commit: 236dc594e15792d1b53501b7595edd571993e03b
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 5 02:52:40 2018 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Mon Feb 5 02:52:40 2018 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=236dc594
use perform_checksum for checksum on files
pym/tbc/build_job.py | 3 ++-
pym/tbc/build_log.py | 4 ++--
pym/tbc/check_setup.py | 5 +++--
pym/tbc/package.py | 3 ++-
pym/tbc/updatedb.py | 3 ++-
5 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/pym/tbc/build_job.py b/pym/tbc/build_job.py
index d0d72c3..8d10a68 100644
--- a/pym/tbc/build_job.py
+++ b/pym/tbc/build_job.py
@@ -12,6 +12,7 @@ from portage import _encodings
from portage import _unicode_decode
from portage.versions import cpv_getkey
from portage.dep import check_required_use
+from portage.checksum import perform_checksum
from tbc.depclean import do_depclean
from tbc.flags import tbc_use_flags
from tbc.qachecks import check_file_in_manifest
@@ -34,7 +35,7 @@ class build_job_action(object):
pkgdir = portdb.getRepositoryPath(repo) + "/" + cp
build_use_flags_list = []
try:
- ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir + "/" + package + "-" + build_dict['ebuild_version'] + ".ebuild")[0]
+ ebuild_version_checksum_tree = perform_checksum(pkgdir + "/" + package + "-" + build_dict['ebuild_version'] + ".ebuild", "SHA256")[0]
except:
ebuild_version_checksum_tree = None
if ebuild_version_checksum_tree == build_dict['checksum']:
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 762ebd4..e9e5dd0 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -14,7 +14,7 @@ from portage.util import writemsg, \
writemsg_level, writemsg_stdout
from portage import _encodings
from portage import _unicode_encode
-
+from portage.checksum import perform_checksum
from _emerge.main import parse_opts
portage.proxy.lazyimport.lazyimport(globals(),
@@ -106,7 +106,7 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
else:
build_dict['build_useflags'] = use_flagsDict
pkgdir = myportdb.getRepositoryPath(repo) + "/" + categories + "/" + package
- ebuild_version_checksum_tree = portage.checksum.sha256hash(pkgdir+ "/" + package + "-" + ebuild_version + ".ebuild")[0]
+ ebuild_version_checksum_tree = perform_checksum(pkgdir+ "/" + package + "-" + ebuild_version + ".ebuild", "SHA256")[0]
build_dict['checksum'] = ebuild_version_checksum_tree
ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id'], build_dict['ebuild_version'])
if status:
diff --git a/pym/tbc/check_setup.py b/pym/tbc/check_setup.py
index a5f53e7..dcd7a0c 100644
--- a/pym/tbc/check_setup.py
+++ b/pym/tbc/check_setup.py
@@ -7,6 +7,7 @@ import os
import errno
from portage.exception import DigestException, FileNotFound, ParseError, PermissionDenied
+from portage.checksum import perform_checksum
from tbc.text import get_file_text
from tbc.sqlquerys import get_config_all_info, add_logs, get_configmetadata_info, get_setup_info
from tbc.sync import git_pull
@@ -29,7 +30,7 @@ def check_make_conf(session, config_id):
# Check if we have some error in the file. (portage.util.getconfig)
# Check if we envorment error with the config. (settings.validate)
try:
- make_conf_checksum_tree = portage.checksum.sha256hash(make_conf_file)[0]
+ make_conf_checksum_tree = perform_checksum(make_conf_file, "SHA256")[0]
portage.util.getconfig(make_conf_file, tolerant=0, allow_sourcing=True, expand=True)
mysettings = portage.config(config_root = check_config_dir)
mysettings.validate()
@@ -61,7 +62,7 @@ def check_configure_guest(session, config_id):
# Check if we have some error in the file (portage.util.getconfig)
# Check if we envorment error with the config (settings.validate)
try:
- make_conf_checksum_tree = portage.checksum.sha256hash(make_conf_file)[0]
+ make_conf_checksum_tree = perform_checksum(make_conf_file, "SHA256")[0]
portage.util.getconfig(make_conf_file, tolerant=0, allow_sourcing=True, expand=True)
mysettings = portage.config(config_root = "/")
mysettings.validate()
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 721f9b1..84e3907 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -9,6 +9,7 @@ import git
import portage
import datetime
from portage.xml.metadata import MetaDataXML
+from portage.checksum import perform_checksum
from tbc.flags import tbc_use_flags
from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
from tbc.flags import tbc_use_flags
@@ -111,7 +112,7 @@ class tbc_package(object):
ebuild_file = pkgdir + "/" + package + "-" + ebuild_version_tree + ".ebuild"
# Make a checksum of the ebuild
try:
- ebuild_version_checksum_tree = portage.checksum.sha256hash(ebuild_file)[0]
+ ebuild_version_checksum_tree = perform_checksum(ebuild_file, "SHA256")[0]
except:
ebuild_version_checksum_tree = "0"
log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,)
diff --git a/pym/tbc/updatedb.py b/pym/tbc/updatedb.py
index 437cfd5..d32f0ac 100644
--- a/pym/tbc/updatedb.py
+++ b/pym/tbc/updatedb.py
@@ -9,6 +9,7 @@ import time
import re
import portage
from portage.xml.metadata import MetaDataXML
+from portage.checksum import perform_checksum
from sqlalchemy.orm import scoped_session, sessionmaker
from tbc.ConnectionManager import NewConnection
from tbc.sqlquerys import get_package_info, update_repo_db, \
@@ -46,7 +47,7 @@ def get_categories_metadataDict(pkgdir):
pkg_md = MetaDataXML(pkgdir + "/metadata.xml", None)
metadata_xml_descriptions_tree = re.sub('\t', '', pkg_md.descriptions()[0])
categories_metadataDict['metadata_xml_descriptions'] = re.sub('\n', '', metadata_xml_descriptions_tree)
- categories_metadataDict['metadata_xml_checksum'] = portage.checksum.sha256hash(pkgdir + "/metadata.xml")[0]
+ categories_metadataDict['metadata_xml_checksum'] = perform_checksum(pkgdir + "/metadata.xml", "SHA256")[0]
return categories_metadataDict
def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings, config_id):
^ permalink raw reply related [flat|nested] 45+ messages in thread
end of thread, other threads:[~2018-02-05 2:53 UTC | newest]
Thread overview: 45+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2016-02-02 23:13 [gentoo-commits] proj/tinderbox-cluster:master commit in: pym/tbc/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2018-02-05 2:52 Magnus Granberg
2017-01-15 19:57 Magnus Granberg
2017-01-08 15:33 Magnus Granberg
2017-01-08 15:33 Magnus Granberg
2016-12-22 18:31 Magnus Granberg
2016-12-22 11:29 Magnus Granberg
2016-12-22 11:29 Magnus Granberg
2016-12-22 11:29 Magnus Granberg
2016-12-22 10:15 Magnus Granberg
2016-12-22 10:11 Magnus Granberg
2016-12-22 10:11 Magnus Granberg
2016-12-20 22:13 Magnus Granberg
2016-12-20 21:21 Magnus Granberg
2016-12-20 21:16 Magnus Granberg
2016-12-20 20:25 Magnus Granberg
2016-12-20 20:25 Magnus Granberg
2016-12-20 20:25 Magnus Granberg
2016-12-10 16:17 Magnus Granberg
2016-12-05 21:11 Magnus Granberg
2016-11-20 15:37 Magnus Granberg
2016-11-15 21:58 Magnus Granberg
2016-03-06 17:47 Magnus Granberg
2016-03-03 15:26 Magnus Granberg
2016-03-02 20:50 Magnus Granberg
2016-02-29 23:01 Magnus Granberg
2016-02-22 18:33 Magnus Granberg
2016-02-22 18:11 Magnus Granberg
2016-02-22 17:08 Magnus Granberg
2016-02-20 12:12 Magnus Granberg
2016-02-10 20:24 Magnus Granberg
2016-01-31 13:39 Magnus Granberg
2016-01-27 5:21 Magnus Granberg
2016-01-26 23:16 Magnus Granberg
2016-01-26 20:29 Magnus Granberg
2016-01-25 22:48 Magnus Granberg
2016-01-18 20:26 Magnus Granberg
2016-01-10 12:53 Magnus Granberg
2016-01-07 7:33 Magnus Granberg
2015-12-21 22:54 Magnus Granberg
2015-12-20 0:12 Magnus Granberg
2015-10-06 19:31 Magnus Granberg
2015-09-06 20:29 Magnus Granberg
2015-09-03 21:03 Magnus Granberg
2015-08-05 1:10 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox