* [gentoo-commits] proj/tinderbox-cluster:master commit in: bin/, pym/tbc/
@ 2016-01-24 22:23 Magnus Granberg
0 siblings, 0 replies; 2+ messages in thread
From: Magnus Granberg @ 2016-01-24 22:23 UTC (permalink / raw
To: gentoo-commits
commit: 070938ef89e8cc93fe2072c540d8e620607b2bee
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sun Jan 24 22:24:26 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sun Jan 24 22:24:26 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=070938ef
add support for logging
bin/tbc_guest_jobs | 30 +++++++++++++++++++++--------
bin/tbc_host_jobs | 14 +++++++++-----
pym/tbc/build_log.py | 22 ++++++++++++++--------
pym/tbc/jobs.py | 31 +++++++++++++++---------------
pym/tbc/log.py | 38 +++++++++++++++++++++++++++++++++++++
pym/tbc/package.py | 53 ++++++++++++++++++++++++++--------------------------
pym/tbc/readconf.py | 10 ++++++++--
pym/tbc/sqlquerys.py | 9 ++++++---
pym/tbc/sync.py | 21 +++++++++++++--------
pym/tbc/updatedb.py | 21 +++++++++++----------
10 files changed, 164 insertions(+), 85 deletions(-)
diff --git a/bin/tbc_guest_jobs b/bin/tbc_guest_jobs
index 721c7e1..b4b74fd 100755
--- a/bin/tbc_guest_jobs
+++ b/bin/tbc_guest_jobs
@@ -1,15 +1,16 @@
-#!/usr/bin/python
+#!/usr/bin/python3.4
#
-# Copyright 1998-2015 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function
from tbc.readconf import read_config_settings
from tbc.ConnectionManager import NewConnection
-from tbc.sqlquerys import add_logs, get_config_id_fqdn, check_host_updatedb, update_deamon_status
+from tbc.sqlquerys import get_config_id_fqdn, check_host_updatedb, update_deamon_status
from tbc.check_setup import check_configure_guest
from tbc.build_job import build_job_action
from tbc.jobs import jobs_main
+from tbc.log import setup_logger, write_log
from sqlalchemy.orm import sessionmaker
import portage
import sys
@@ -19,11 +20,18 @@ import time
def main():
repeat = True
tbc_settings = read_config_settings()
+
+ # setupt the logger
+ logger = setup_logger(tbc_settings):
+
Session = sessionmaker(bind=NewConnection(tbc_settings))
session = Session()
config_id = get_config_id_fqdn(session, tbc_settings['hostname'])
- add_logs(session, "Job and build deamon started.", "info", config_id)
- update_deamon_status(session, 'Waiting', config_id)
+ write_log(session, "Job and build deamon started.", "info", config_id, 'main')
+ Status = 'Waiting'
+ update_deamon_status(session, Status, config_id)
+ msg = 'Status: %s Host: %s' % (Status, tbc_settings['hostname'],)
+ write_log(session, msg, "info", config_id, 'main')
init_build_job = build_job_action(config_id, session)
while repeat:
jobs_main(session, config_id)
@@ -31,10 +39,16 @@ def main():
time.sleep(60)
continue
else:
- update_deamon_status(session, 'Runing', config_id)
+ Status = 'Runing'
+ update_deamon_status(session, Status, config_id)
+ msg = 'Status: %s Host: %s' % (Status, tbc_settings['hostname'],)
+ write_log(session, msg, "info", config_id, 'main')
init_build_job.procces_build_jobs()
- time.sleep(5)
- update_deamon_status(session, 'Waiting', config_id)
+ Status = 'Waiting'
+ update_deamon_status(session, Status, config_id)
+ msg = 'Status: %s Host: %s' % (Status, tbc_settings['hostname'],)
+ write_log(session, msg, "info", config_id, 'main')
+ time.sleep(60)
conn.close
if __name__ == "__main__":
diff --git a/bin/tbc_host_jobs b/bin/tbc_host_jobs
index c9bbf8c..629ef54 100755
--- a/bin/tbc_host_jobs
+++ b/bin/tbc_host_jobs
@@ -1,6 +1,6 @@
-#!/usr/bin/python
+#!/usr/bin/python3.4
#
-# Copyright 1998-2015 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function
@@ -8,23 +8,27 @@ from __future__ import print_function
from tbc.readconf import read_config_settings
from tbc.jobs import jobs_main
from tbc.ConnectionManager import NewConnection
-from tbc.sqlquerys import add_logs, get_config_id_fqdn
+from tbc.sqlquerys import get_config_id_fqdn
+from tbc.log import setup_logger, write_log
from sqlalchemy.orm import sessionmaker
import time
def main():
# Main
tbc_settings = read_config_settings()
+ # setupt the logger
+ logger = setup_logger(tbc_settings):
+
Session = sessionmaker(bind=NewConnection(tbc_settings))
session = Session()
config_id = get_config_id_fqdn(session, tbc_settings['hostname'])
- add_logs(session, "Job deamon started", "info", config_id)
+ write_log(session, "Job deamon started", "info", config_id, 'main')
repeat = True
while repeat:
jobs_main(session, config_id)
repeat = False
time.sleep(60)
- add_logs(session, "Job deamon stoped", "info", config_id)
+ write_log(session, "Job deamon stoped", "info", config_id, 'main')
if __name__ == "__main__":
main()
\ No newline at end of file
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index 7b3784a..70cd091 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2015 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function
@@ -6,6 +6,7 @@ import re
import os
import platform
import hashlib
+import logging
from portage.versions import catpkgsplit, cpv_getversion
import portage
@@ -25,15 +26,18 @@ from tbc.text import get_log_text_dict
from tbc.readconf import read_config_settings
from tbc.flags import tbc_use_flags
from tbc.ConnectionManager import NewConnection
-from tbc.sqlquerys import add_logs, get_config_id, get_ebuild_id_db, add_new_buildlog, \
+from tbc.sqlquerys import get_config_id, get_ebuild_id_db, add_new_buildlog, \
get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \
add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \
update_buildjobs_status, update_mtime_sql, add_repoman_qa, get_config_id_fqdn, get_setup_info, \
add_repoman_log
+from tbc.log import write_log
+
from sqlalchemy.orm import sessionmaker
def check_repoman_full(session, pkgdir, package_id, config_id, cpv=False):
# Check cp with repoman repoman full
+ write_log(session, 'Repoman Check', "info", config_id, 'build_log.check_repoman_full')
status = repoman_full(session, pkgdir, config_id)
repoman_hash = hashlib.sha256()
if cpv:
@@ -59,7 +63,9 @@ def check_repoman_full(session, pkgdir, package_id, config_id, cpv=False):
repoman_log = repoman_log + line + "\n"
repoman_hash.update(line.encode('utf-8'))
add_repoman_log(session, package_id, repoman_log, repoman_hash.hexdigest())
+ write_log(session, 'Repoman Check Fail\n' + repoman_log, "warning", config_id, 'build_log.check_repoman_full')
return repoman_log
+ write_log(session, 'Repoman Check Pass', "info", config_id, 'build_log.check_repoman_full')
return False
def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
@@ -281,7 +287,7 @@ def add_buildlog_main(settings, pkg, trees):
build_dict = get_build_dict_db(session, config_id, settings, tbc_settings, pkg)
if build_dict is None:
log_msg = "Package %s:%s is NOT logged." % (pkg.cpv, pkg.repo,)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'build_log.add_buildlog_main')
session.close
return
build_log_dict = {}
@@ -298,19 +304,18 @@ def add_buildlog_main(settings, pkg, trees):
build_log_dict['log_hash'] = log_hash.hexdigest()
build_log_dict['logfilename'] = settings.get("PORTAGE_LOG_FILE").split(host_config)[1]
log_msg = "Logfile name: %s" % (settings.get("PORTAGE_LOG_FILE"),)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'build_log.add_buildlog_main')
build_log_dict['emerge_info'] = get_emerge_info_id(settings, trees, session, config_id)
log_id = add_new_buildlog(session, build_dict, build_log_dict)
if log_id is None:
log_msg = "Package %s:%s is NOT logged." % (pkg.cpv, pkg.repo,)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'build_log.add_buildlog_main')
else:
add_repoman_qa(session, build_log_dict, log_id)
os.chmod(settings.get("PORTAGE_LOG_FILE"), 0o664)
log_msg = "Package: %s:%s is logged." % (pkg.cpv, pkg.repo,)
- add_logs(session, log_msg, "info", config_id)
- print("\n>>> Logging %s:%s\n" % (pkg.cpv, pkg.repo,))
+ write_log(session, log_msg, "info", config_id, 'build_log.add_buildlog_main')
build_msg = "BUILD: PASS"
qa_msg = "QA: PASS"
repoman_msg = "REPOMAN: PASS"
@@ -322,7 +327,8 @@ def add_buildlog_main(settings, pkg, trees):
qa_msg = "QA: FAILD"
else:
build_msg = "BUILD: FAILD"
- msg = "Package: %s Repo: %s %s %s %s Weblink http://foo.gg.oo/new/logs/build/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, log_id,)
+ msg = "Package: %s Repo: %s %s %s %s Weblink http://77.110.8.67/new/logs/build/%s\n" % (pkg.cpv, pkg.repo, build_msg, repoman_msg, qa_msg, log_id,)
+ write_log(session, msg, "info", config_id, 'build_log.add_buildlog_main')
send_irk(msg)
session.close
diff --git a/pym/tbc/jobs.py b/pym/tbc/jobs.py
index 924421c..5c136d2 100644
--- a/pym/tbc/jobs.py
+++ b/pym/tbc/jobs.py
@@ -1,11 +1,12 @@
-# Copyright 1998-2015 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function
from tbc.sync import git_sync_main
#from tbc.buildquerydb import add_buildquery_main, del_buildquery_main
from tbc.updatedb import update_db_main
-from tbc.sqlquerys import get_config_id, add_logs, get_jobs, update_job_list
+from tbc.sqlquerys import get_config_id, get_jobs, update_job_list
+from tbc.log import write_log
def jobs_main(session, config_id):
JobsInfo = get_jobs(session, config_id)
@@ -16,55 +17,55 @@ def jobs_main(session, config_id):
run_config_id = JobInfo.RunConfigId
job_id = JobInfo.JobId
log_msg = "Job: %s Type: %s" % (job_id, job,)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'jobs_main')
if job == "addbuildquery":
update_job_list(session, "Runing", job_id)
log_msg = "Job %s is runing." % (job_id,)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'jobs_main')
#result = add_buildquery_main(run_config_id)
#if result is True:
# update_job_list(session, "Done", job_id)
# log_msg = "Job %s is done.." % (job_id,)
- # add_logs(session, log_msg, "info", config_id)
+ # write_log(session, log_msg, "info", config_id, 'jobs_main')
#else:
# update_job_list(session, "Fail", job_id)
# log_msg = "Job %s did fail." % (job_id,)
- # add_logs(session, log_msg, "info", config_id)
+ # write_log(session, log_msg, "info", config_id, 'jobs_main')
elif job == "delbuildquery":
update_job_list(session, "Runing", job_id)
log_msg = "Job %s is runing." % (job_id,)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'jobs_main')
#result = del_buildquery_main(config_id)
#if result is True:
# update_job_list(session, "Done", job_id)
# log_msg = "Job %s is done.." % (job_id,)
- # add_logs(session, log_msg, "info", config_id)
+ # write_log(session, log_msg, "info", config_id, 'jobs_main')
#else:
# update_job_list(session, "Fail", job_id)
# log_msg = "Job %s did fail." % (job_id,)
- # add_logs(session, log_msg, "info", config_id)
+ # write_log(session, log_msg, "info", config_id, 'jobs_main')
elif job == "esync":
update_job_list(session, "Runing", job_id)
log_msg = "Job %s is runing." % (job_id,)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'jobs_main')
if update_db_main(session, git_sync_main(session), config_id):
update_job_list(session, "Done", job_id)
log_msg = "Job %s is done.." % (job_id,)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'jobs_main')
else:
update_job_list(session, "Fail", job_id)
log_msg = "Job %s did fail." % (job_id,)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'jobs_main')
elif job == "updatedb":
update_job_list(session, "Runing", job_id)
log_msg = "Job %s is runing." % (job_id,)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'jobs_main')
if update_db_main(session, None, config_id):
update_job_list(session, "Done", job_id)
log_msg = "Job %s is done.." % (job_id,)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'jobs_main')
else:
update_job_list(session, "Fail", job_id)
log_msg = "Job %s did fail." % (job_id,)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'jobs_main')
return
diff --git a/pym/tbc/log.py b/pym/tbc/log.py
new file mode 100644
index 0000000..944d3fa
--- /dev/null
+++ b/pym/tbc/log.py
@@ -0,0 +1,38 @@
+# Copyright 1998-2016 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import print_function
+import logging
+module_logger = logging.getLogger('tbc.log')
+
+def setup_logger( tbc_settings):
+ # setupt the logger
+ log_level = getattr(logging, tbc_settings['log_level'].upper(), None)
+ format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+ if not isinstance(log_level, int):
+ raise ValueError('Invalid log level: %s' % tbc_settings['log_level'])
+ logging.basicConfig()
+ logger = logging.getLogger('tbc')
+ logger.setLevel(log_level)
+ if tbc_settings['log_file']:
+ fh = logging.FileHandler(tbc_settings['log_file'])
+ else:
+ fh = logging.StreamHandler()
+ formatter = logging.Formatter(format)
+ fh.setFormatter(formatter)
+ logger.addHandler(fh)
+ return logger
+
+def write_log(session, msg, level, config_id, function=False):
+ if function:
+ logger = logging.getLogger('tbc.' + function)
+ else:
+ logger = logging.getLogger('tbc')
+ if level == 'info':
+ logger.info(msg)
+ if level == 'error':
+ logger.error(msg)
+ if level == 'debug':
+ logger.debug(msg)
+ if level == 'warning':
+ logger.warning(msg)
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 40de495..72108a2 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -13,7 +13,8 @@ from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
from tbc.flags import tbc_use_flags
from tbc.qachecks import check_repoman
from tbc.build_log import check_repoman_full
-from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
+from tbc.log import write_log
+from tbc.sqlquerys import get_package_info, get_config_info, \
add_new_build_job, add_new_ebuild_sql, get_ebuild_id_list, add_old_ebuild, \
get_package_metadata_sql, update_package_metadata, update_mtime_sql, \
get_package_info_from_package_id, get_config_all_info, add_new_package_sql, \
@@ -104,9 +105,9 @@ class tbc_package(object):
except:
ebuild_version_checksum_tree = "0"
log_msg = "QA: Can't checksum the ebuild file. %s on repo %s" % (cpv, repo,)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "warning", config_id, 'packages.get_packageDict')
log_msg = "C %s:%s ... Fail." % (cpv, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "info", config_id, 'packages.get_packageDict')
git_commit = '0'
else:
git_commit = self.get_git_log_ebuild(repodir, ebuild_file)
@@ -118,7 +119,7 @@ class tbc_package(object):
# so it can be updated next time we update the db
if not ebuild_version_metadata_tree:
log_msg = " QA: %s have broken metadata on repo %s" % (cpv, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "warning", config_id, 'packages.get_packageDict')
ebuild_version_metadata_tree = ['','','','','','','','','','','','','','','','','','','','','','','','','']
ebuild_version_checksum_tree = '0'
@@ -167,7 +168,7 @@ class tbc_package(object):
# B = Build cpv use-flags config
# FIXME log_msg need a fix to log the use flags corect.
log_msg = "B %s:%s USE: %s Setup: %s" % (k, v['repo'], use_flagsDict, setup_id,)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "info", config_id, 'packages.add_new_build_job_db')
i = i +1
def get_git_log_pkg_text(self, repodir, cp):
@@ -188,7 +189,7 @@ class tbc_package(object):
pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
except:
log_msg = "Metadata file %s is missing or has errors" % (pkgdir + "/metadata.xml")
- add_logs(self._session, log_msg, "qa", self._config_id)
+ write_log(session, log_msg, "warning", config_id, 'packages.get_package_metadataDict')
else:
tmp_herds = pkg_md.herds()
if tmp_herds != ():
@@ -200,7 +201,7 @@ class tbc_package(object):
attDict['metadata_xml_email'] = md_email_list
else:
log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
- add_logs(self._session, log_msg, "qa", self._config_id)
+ write_log(session, log_msg, "warning", config_id, 'packages.get_package_metadataDict')
attDict['git_log_pkg_text'] = self.get_git_log_pkg_text(repodir, cp)
attDict['metadata_xml_descriptions'] = ''
package_metadataDict[package_id] = attDict
@@ -246,9 +247,9 @@ class tbc_package(object):
mtime = os.path.getmtime(pkgdir + "/Manifest")
except:
log_msg = "QA: Can't checksum the Manifest file. :%s:%s" % (cp, repo,)
- add_logs(self._session, log_msg, "error", self._config_id)
+ write_log(session, log_msg, "warning", config_id, 'packages.get_manifest_mtime_tree')
log_msg = "C %s:%s ... Fail." % (cp, repo)
- add_logs(self._session, log_msg, "error", self._config_id)
+ write_log(session, log_msg, "warning", config_id, 'packages.get_manifest_mtime_tree')
return False
return datetime.datetime.fromtimestamp(mtime).replace(microsecond=0)
@@ -257,9 +258,9 @@ class tbc_package(object):
# C = Checking
# N = New Package
log_msg = "C %s:%s" % (cp, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "info", config_id, 'packages.add_new_package_db')
log_msg = "N %s:%s" % (cp, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "info", config_id, 'packages.add_new_package_db')
repodir = self._myportdb.getRepositoryPath(repo)
mytree = []
mytree.append(repodir)
@@ -274,16 +275,16 @@ class tbc_package(object):
status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
if status:
log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
- add_logs(self._session, log_msg, "error", self._config_id)
+ write_log(session, log_msg, "warning", config_id, 'packages.add_new_package_db')
package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
# Get the ebuild list for cp
ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
if ebuild_list_tree == []:
log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "error", config_id, 'packages.add_new_package_db')
log_msg = "C %s:%s ... Fail." % (cp, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "warning", config_id, 'packages.add_new_package_db')
return None
# Make the needed packageDict with ebuild infos so we can add it later to the db.
@@ -300,7 +301,7 @@ class tbc_package(object):
packageDict[cpv]['new'] = True
self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "info", config_id, 'packages.add_new_package_db')
def update_package_db(self, package_id):
# Update the categories and package with new info
@@ -309,7 +310,7 @@ class tbc_package(object):
cp = CategoryInfo.Category + '/' + PackageInfo.Package
repo = RepoInfo.Repo
log_msg = "C %s:%s" % (cp, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
repodir = self._myportdb.getRepositoryPath(repo)
pkgdir = repodir + "/" + cp # Get RepoDIR + cp
if not os.path.isdir(pkgdir):
@@ -319,11 +320,11 @@ class tbc_package(object):
cpv = cp + "-" + EbuildInfo.Version
# R = remove ebuild
log_msg = "R %s:%s" % (cpv, repo,)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
add_old_ebuild(self._session, old_ebuild_id_list)
add_old_package(self._session, package_id)
log_msg = "C %s:%s ... Done." % (cp, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
return None
mytree = []
@@ -337,23 +338,23 @@ class tbc_package(object):
# U = Update
log_msg = "U %s:%s" % (cp, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
# Get the ebuild list for cp
old_ebuild_id_list = []
ebuild_list_tree = self._myportdb.cp_list(cp, use_cache=1, mytree=mytree)
if ebuild_list_tree == []:
log_msg = "QA: Can't get the ebuilds list. %s:%s" % (cp, repo,)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "error", config_id, 'packages.update_package_db')
log_msg = "C %s:%s ... Fail." % (cp, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "warning", config_id, 'packages.update_package_db')
return None
# Check cp with repoman full
status = check_repoman_full(self._session, pkgdir, package_id, self._config_id)
if status:
log_msg = "Repoman %s::%s ... Fail." % (cp, repo)
- add_logs(self._session, log_msg, "error", self._config_id)
+ write_log(session, log_msg, "warning", config_id, 'packages.update_package_db')
package_metadataDict = self.get_package_metadataDict(pkgdir, package_id)
packageDict ={}
new_ebuild_id_list = []
@@ -382,7 +383,7 @@ class tbc_package(object):
ebuilds_id , status = get_ebuild_id_db(self._session, checksum, package_id)
for ebuild_id in ebuilds_id:
log_msg = "U %s:%s:%s Dups of checksums" % (cpv, repo, ebuild_id,)
- add_logs(self._session, log_msg, "error", self._config_id)
+ write_log(session, log_msg, "warning", config_id, 'packages.update_package_db')
dupe_ebuild_id_list.append(ebuild_id)
add_old_ebuild(self._session, dupe_ebuild_id_list)
ebuild_version_manifest_checksum_db = None
@@ -393,12 +394,12 @@ class tbc_package(object):
if ebuild_version_manifest_checksum_db is None:
# N = New ebuild
log_msg = "N %s:%s" % (cpv, repo,)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
packageDict[cpv]['new'] = True
elif ebuild_version_checksum_tree != ebuild_version_manifest_checksum_db:
# U = Updated ebuild
log_msg = "U %s:%s" % (cpv, repo,)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
else:
# Remove cpv from packageDict and add ebuild to new ebuils list
del packageDict[cpv]
@@ -407,4 +408,4 @@ class tbc_package(object):
self.add_package(packageDict, package_metadataDict, package_id, new_ebuild_id_list, old_ebuild_id_list, manifest_mtime_tree)
log_msg = "C %s:%s ... Done." % (cp, repo)
- add_logs(self._session, log_msg, "info", self._config_id)
+ write_log(session, log_msg, "info", config_id, 'packages.update_package_db')
diff --git a/pym/tbc/readconf.py b/pym/tbc/readconf.py
index b4bad11..ebb3b71 100644
--- a/pym/tbc/readconf.py
+++ b/pym/tbc/readconf.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2015 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
import os
@@ -10,6 +10,8 @@ configfile = "/etc/tbc/tbc.conf"
def read_config_settings():
# It will return a dict with options from the configfile
+ log_file= False
+ tbc_settings = {}
try:
open_conffile = open(configfile, 'r')
except:
@@ -27,13 +29,17 @@ def read_config_settings():
get_sql_user = element[1]
if element[0] == 'SQLPASSWD': # Password
get_sql_passwd = element[1]
+ if element[0] == 'LOG': # Log level
+ tbc_settings['log_level'] = element[1].rstrip('\n')
+ if element[0] == 'LOGFILE': # Log level
+ log_file = element[1].rstrip('\n')
open_conffile.close()
- tbc_settings = {}
tbc_settings['sql_backend'] = get_sql_backend.rstrip('\n')
tbc_settings['sql_db'] = get_sql_db.rstrip('\n')
tbc_settings['sql_host'] = get_sql_host.rstrip('\n')
tbc_settings['sql_user'] = get_sql_user.rstrip('\n')
tbc_settings['sql_passwd'] = get_sql_passwd.rstrip('\n')
tbc_settings['hostname'] = getfqdn()
+ tbc_settings['log_file'] = log_file
return tbc_settings
diff --git a/pym/tbc/sqlquerys.py b/pym/tbc/sqlquerys.py
index e93c96d..5100bec 100644
--- a/pym/tbc/sqlquerys.py
+++ b/pym/tbc/sqlquerys.py
@@ -1,4 +1,4 @@
-# Copyright 1998-2015 Gentoo Foundation
+# Copyright 1998-2016 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
from __future__ import print_function
@@ -10,6 +10,7 @@ from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Pack
BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \
Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsRepomanQa, CategoriesMetadata, \
PackagesRepoman
+from tbc.log import write_log
from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
from sqlalchemy import and_, or_
@@ -83,8 +84,6 @@ def get_packages_to_build(session, config_id):
elif BuildJobsTmp.filter_by(BuildNow = False).all() != []:
BuildJobsInfo = BuildJobsTmp.filter_by(BuildNow = False).first()
else:
- log_msg = "BuildJobsTmp found job but the if state mant did not."
- add_zobcs_logs(session, log_msg, "error", config_id)
return None
update_buildjobs_status(session, BuildJobsInfo.BuildJobId, 'Looked', config_id)
EbuildsInfo = session.query(Ebuilds).filter_by(EbuildId = BuildJobsInfo.EbuildId).one()
@@ -219,6 +218,8 @@ def add_new_buildlog(session, build_dict, build_log_dict):
else:
for use in use_list:
useflagsdict[use.UseId] = use.Status
+ msg = 'Log_hash: %s Log_hash_sql: %s Build_log_id: %s' % (build_log_dict['log_hash'], log_hash[0], build_log_id,)
+ write_log(session, msg, "debug", build_dict['config_id'], 'sqlquerys.add_new_buildlog.build_log_id_match')
if log_hash[0] == build_log_dict['log_hash'] and build_dict['build_useflags'] == useflagsdict:
if session.query(BuildLogsConfig).filter(BuildLogsConfig.ConfigId.in_([build_dict['config_id']])).filter_by(BuildLogId = build_log_id[0]):
return None, True
@@ -255,6 +256,8 @@ def add_new_buildlog(session, build_dict, build_log_dict):
session.commit()
return build_log_id
+ msg = 'build_job_id: %s build_log_id_list: %s' % (build_dict['build_job_id'], build_log_id_list,)
+ write_log(session, msg, "debug", build_dict['config_id'], 'sqlquerys.add_new_buildlog')
if build_dict['build_job_id'] is None and build_log_id_list == []:
build_log_id = build_log_id_no_match(build_dict, build_log_dict)
return build_log_id
diff --git a/pym/tbc/sync.py b/pym/tbc/sync.py
index 610f357..1732f1f 100644
--- a/pym/tbc/sync.py
+++ b/pym/tbc/sync.py
@@ -13,6 +13,7 @@ import git
from tbc.sqlquerys import get_config_id_fqdn, add_logs, get_config_all_info, \
get_configmetadata_info, get_config_info, get_setup_info
from tbc.readconf import read_config_settings
+from tbc.log import write_log
def git_repos_list(myportdb):
repo_trees_list = myportdb.porttrees
@@ -46,7 +47,7 @@ def git_sync_main(session):
myportdb = portage.portdbapi(mysettings=mysettings)
GuestBusy = True
log_msg = "Waiting for Guest to be idel"
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'sync.git_sync_main')
guestid_list = []
# check if the guests is idel
for config in get_config_all_info(session):
@@ -57,6 +58,7 @@ def git_sync_main(session):
for guest_id in guestid_list:
ConfigMetadataGuest = get_configmetadata_info(session, guest_id)
Status_list.append(ConfigMetadataGuest.Status)
+ write_log(session, 'Guset status: %s' % (Status_list,), "debug", config_id, 'sync.git_sync_main')
if not 'Runing' in Status_list:
GuestBusy = False
else:
@@ -75,13 +77,15 @@ def git_sync_main(session):
reponame = myportdb.getRepositoryName(repo_dir)
repo = git.Repo(repo_dir)
log_msg = "Checking repo %s" % (reponame)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'sync.git_sync_main')
info_list, repouptodate = git_fetch(repo)
if not repouptodate:
cp_list = []
attr = {}
# We check for Manifest changes and add the package to a list
- for diff_line in repo.git.diff('origin', '--name-only').splitlines():
+ repo_diff = repo.git.diff('origin', '--name-only')
+ write_log(session, 'Git diff: %s' % (repo_diff,), "debug", config_id, 'sync.git_sync_main')
+ for diff_line in repo_diff.splitlines():
if re.search("Manifest$", diff_line):
cp = re.sub('/Manifest', '', diff_line)
cp_list.append(cp)
@@ -90,9 +94,9 @@ def git_sync_main(session):
git_merge(repo, info_list[0])
else:
log_msg = "Repo %s is up to date" % (reponame)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'sync.git_sync_main')
log_msg = "Checking repo %s Done" % (reponame)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'sync.git_sync_main')
# Need to add a clone of profiles/base for reading the tree
try:
os.mkdir(mysettings['PORTDIR'] + "/profiles/config", 0o777)
@@ -103,16 +107,17 @@ def git_sync_main(session):
pass
log_msg = "Repo sync ... Done."
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'sync.git_sync_main')
+ write_log(session, 'Updated Packages: %s' % (repo_cp_dict,), "debug", config_id, 'sync.git_sync_main')
return repo_cp_dict
def git_pull(session, repo_dir, config_id):
log_msg = "Git pull"
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'sync.git_pull')
repo = git.Repo(repo_dir)
info_list, repouptodate = git_fetch(repo)
if not repouptodate:
git_merge(repo, info_list[0])
log_msg = "Git pull ... Done"
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'sync.git_pull')
return True
diff --git a/pym/tbc/updatedb.py b/pym/tbc/updatedb.py
index b860f51..0753d92 100644
--- a/pym/tbc/updatedb.py
+++ b/pym/tbc/updatedb.py
@@ -11,19 +11,20 @@ import portage
from portage.xml.metadata import MetaDataXML
from sqlalchemy.orm import scoped_session, sessionmaker
from tbc.ConnectionManager import NewConnection
-from tbc.sqlquerys import add_logs, get_package_info, update_repo_db, \
+from tbc.sqlquerys import get_package_info, update_repo_db, \
update_categories_db, get_configmetadata_info, get_config_all_info, add_new_build_job, \
get_config_info, get_setup_info
from tbc.check_setup import check_make_conf
from tbc.package import tbc_package
# Get the options from the config file tbc.conf
from tbc.readconf import read_config_settings
+from tbc.log import write_log
def init_portage_settings(session, config_id):
# check config setup
check_make_conf(session, config_id)
log_msg = "Check configs done"
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'updatedb.init_portage_settings')
# setup default root
ConfigsMetaDataInfo = get_configmetadata_info(session, config_id)
@@ -35,7 +36,7 @@ def init_portage_settings(session, config_id):
# Set config_root (PORTAGE_CONFIGROOT) to default_config_root
mysettings = portage.config(config_root = default_config_root)
log_msg = "Setting default config to: %s" % (host_config,)
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'updatedb.init_portage_settings')
return mysettings
def get_categories_metadataDict(pkgdir):
@@ -78,7 +79,7 @@ def update_cpv_db_pool(mysettings, myportdb, cp, repo, tbc_settings, config_id):
def update_cpv_db(session, repo_cp_dict, config_id, tbc_settings):
GuestBusy = True
log_msg = "Waiting for Guest to be idel"
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'updatedb.update_cpv_db')
guestid_list = []
for config in get_config_all_info(session):
if not config.Host:
@@ -93,7 +94,7 @@ def update_cpv_db(session, repo_cp_dict, config_id, tbc_settings):
time.sleep(30)
log_msg = "Checking categories, package, ebuilds"
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'updatedb.update_cpv_db')
new_build_jobs_list = []
# Setup settings, portdb and pool
@@ -101,8 +102,8 @@ def update_cpv_db(session, repo_cp_dict, config_id, tbc_settings):
myportdb = portage.portdbapi(mysettings=mysettings)
# Use all cores when multiprocessing
- pool_cores = multiprocessing.cpu_count()
- pool = multiprocessing.Pool(processes = pool_cores)
+ #pool_cores = multiprocessing.cpu_count()
+ #pool = multiprocessing.Pool(processes = pool_cores)
# Get packages and repo
if repo_cp_dict is None:
@@ -147,7 +148,7 @@ def update_cpv_db(session, repo_cp_dict, config_id, tbc_settings):
# pool.close()
# pool.join()
log_msg = "Checking categories, package and ebuilds ... done"
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'updatedb.update_cpv_db')
def update_db_main(session, repo_cp_dict, config_id):
# Main
@@ -156,10 +157,10 @@ def update_db_main(session, repo_cp_dict, config_id):
# Logging
tbc_settings = read_config_settings()
log_msg = "Update db started."
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'updatedb.update_db_main')
# Update the cpv db
update_cpv_db(session, repo_cp_dict, config_id, tbc_settings)
log_msg = "Update db ... Done."
- add_logs(session, log_msg, "info", config_id)
+ write_log(session, log_msg, "info", config_id, 'updatedb.update_db_main')
return True
^ permalink raw reply related [flat|nested] 2+ messages in thread* [gentoo-commits] proj/tinderbox-cluster:master commit in: bin/, pym/tbc/
@ 2016-01-26 20:50 Magnus Granberg
0 siblings, 0 replies; 2+ messages in thread
From: Magnus Granberg @ 2016-01-26 20:50 UTC (permalink / raw
To: gentoo-commits
commit: 7dfbe838fc5b944b56ec6af3d87d9f788913747f
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jan 26 20:52:01 2016 +0000
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jan 26 20:52:01 2016 +0000
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=7dfbe838
fix small typos for the looging
bin/tbc_guest_jobs | 4 ++--
bin/tbc_host_jobs | 4 ++--
pym/tbc/build_log.py | 4 ++--
pym/tbc/package.py | 2 +-
4 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/bin/tbc_guest_jobs b/bin/tbc_guest_jobs
index b4b74fd..5614f07 100755
--- a/bin/tbc_guest_jobs
+++ b/bin/tbc_guest_jobs
@@ -21,8 +21,8 @@ def main():
repeat = True
tbc_settings = read_config_settings()
- # setupt the logger
- logger = setup_logger(tbc_settings):
+ # setup the logger
+ logger = setup_logger(tbc_settings)
Session = sessionmaker(bind=NewConnection(tbc_settings))
session = Session()
diff --git a/bin/tbc_host_jobs b/bin/tbc_host_jobs
index 629ef54..50dfe76 100755
--- a/bin/tbc_host_jobs
+++ b/bin/tbc_host_jobs
@@ -16,8 +16,8 @@ import time
def main():
# Main
tbc_settings = read_config_settings()
- # setupt the logger
- logger = setup_logger(tbc_settings):
+ # setup the logger
+ logger = setup_logger(tbc_settings)
Session = sessionmaker(bind=NewConnection(tbc_settings))
session = Session()
diff --git a/pym/tbc/build_log.py b/pym/tbc/build_log.py
index ec4aaa7..d4fa0ef 100644
--- a/pym/tbc/build_log.py
+++ b/pym/tbc/build_log.py
@@ -75,8 +75,8 @@ def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
package = cpvr_list[1]
repo = pkg.repo
ebuild_version = cpv_getversion(pkg.cpv)
- log_msg = "Logging %s:%s" % (pkg.cpv, repo,)
- add_logs(session, log_msg, "info", config_id)
+ log_msg = "Setting up logging for %s:%s" % (pkg.cpv, repo,)
+ write_log(session, log_msg, "info", config_id, 'build_log.get_build_dict_db')
PackageInfo = get_package_info(session, categories, package, repo)
build_dict = {}
build_dict['ebuild_version'] = ebuild_version
diff --git a/pym/tbc/package.py b/pym/tbc/package.py
index 68574cf..bb797de 100644
--- a/pym/tbc/package.py
+++ b/pym/tbc/package.py
@@ -112,7 +112,7 @@ class tbc_package(object):
write_log(self._session, log_msg, "info", self._config_id, 'packages.get_packageDict')
git_commit = '0'
else:
- repodir =self._ myportdb.getRepositoryPath(repo)
+ repodir =self._myportdb.getRepositoryPath(repo)
git_commit = self.get_git_log_ebuild(repodir, ebuild_file)
# Get the ebuild metadata
^ permalink raw reply related [flat|nested] 2+ messages in thread
end of thread, other threads:[~2016-01-26 20:51 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2016-01-24 22:23 [gentoo-commits] proj/tinderbox-cluster:master commit in: bin/, pym/tbc/ Magnus Granberg
-- strict thread matches above, loose matches on Subject: below --
2016-01-26 20:50 Magnus Granberg
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox