public inbox for gentoo-commits@lists.gentoo.org
 help / color / mirror / Atom feed
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/, sql/
@ 2015-05-09 15:03 Magnus Granberg
  0 siblings, 0 replies; 2+ messages in thread
From: Magnus Granberg @ 2015-05-09 15:03 UTC (permalink / raw
  To: gentoo-commits

commit:     f2b047e20193e5cafe00e71846ba84df5fa3cc59
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Sat May  9 14:58:13 2015 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Sat May  9 14:58:13 2015 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=f2b047e2

Add repoman and qa info to db, fix buildjob check and some smaller fix.

 sql/structure_dump.sql | 93 +++++++++++++++++++++++++-------------------------
 tbc/pym/build_log.py   | 24 ++++++-------
 tbc/pym/check_setup.py | 15 +++++---
 tbc/pym/db_mapping.py  | 14 +++++++-
 tbc/pym/jobs.py        | 20 ++---------
 tbc/pym/package.py     |  2 +-
 tbc/pym/readconf.py    |  8 ++---
 tbc/pym/sqlquerys.py   | 35 ++++++++++++++-----
 tbc/pym/sync.py        | 44 ++++++++++++++++++++----
 tbc/pym/updatedb.py    |  2 +-
 10 files changed, 153 insertions(+), 104 deletions(-)

diff --git a/sql/structure_dump.sql b/sql/structure_dump.sql
index 5fe548b..9d9ecba 100644
--- a/sql/structure_dump.sql
+++ b/sql/structure_dump.sql
@@ -3,7 +3,7 @@
 -- http://www.phpmyadmin.net
 --
 -- Host: localhost
--- Generation Time: Apr 18, 2015 at 08:31 PM
+-- Generation Time: May 09, 2015 at 02:34 PM
 -- Server version: 10.0.15-MariaDB-log
 -- PHP Version: 5.6.5-pl0-gentoo
 
@@ -17,7 +17,7 @@ SET time_zone = "+00:00";
 /*!40101 SET NAMES utf8 */;
 
 --
--- Database: `tbc`
+-- Database: `zobcs`
 --
 
 DELIMITER $$
@@ -57,10 +57,10 @@ CREATE TABLE IF NOT EXISTS `build_jobs` (
   `setup_id` int(11) NOT NULL,
   `config_id` int(11) NOT NULL,
   `status` enum('Waiting','Building','Looked') NOT NULL DEFAULT 'Waiting',
-  `build_now` tinyint(1) NOT NULL DEFAULT '0',
+  `build_now` tinyint(1) NOT NULL,
   `removebin` tinyint(1) NOT NULL,
   `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
-) ENGINE=MyISAM AUTO_INCREMENT=1325 DEFAULT CHARSET=utf8 COMMENT='The build work list';
+) ENGINE=MyISAM AUTO_INCREMENT=3113 DEFAULT CHARSET=utf8 COMMENT='The build work list';
 
 -- --------------------------------------------------------
 
@@ -99,7 +99,7 @@ CREATE TABLE IF NOT EXISTS `build_jobs_use` (
   `build_job_id` int(11) NOT NULL,
   `use_id` int(11) NOT NULL,
   `status` tinyint(1) NOT NULL DEFAULT '0'
-) ENGINE=MyISAM AUTO_INCREMENT=6067 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=17343 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -115,7 +115,7 @@ CREATE TABLE IF NOT EXISTS `build_logs` (
   `log_hash` varchar(100) NOT NULL,
   `bug_id` int(10) NOT NULL DEFAULT '0',
   `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
-) ENGINE=MyISAM AUTO_INCREMENT=15154 DEFAULT CHARSET=utf8 COMMENT='Main log info for the builds';
+) ENGINE=MyISAM AUTO_INCREMENT=17625 DEFAULT CHARSET=utf8 COMMENT='Main log info for the builds';
 
 -- --------------------------------------------------------
 
@@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS `build_logs_config` (
   `einfo_id` int(11) NOT NULL,
   `logname` varchar(150) NOT NULL COMMENT 'filename of the log',
   `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
-) ENGINE=MyISAM AUTO_INCREMENT=41689 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=44160 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -154,7 +154,7 @@ CREATE TABLE IF NOT EXISTS `build_logs_errors` (
 `id` int(11) NOT NULL,
   `build_log_id` int(11) NOT NULL,
   `error_id` int(11) NOT NULL
-) ENGINE=MyISAM AUTO_INCREMENT=2616 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=3379 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -168,7 +168,7 @@ CREATE TABLE IF NOT EXISTS `build_logs_hilight` (
   `start_line` int(11) NOT NULL,
   `end_line` int(11) NOT NULL,
   `hilight_css_id` int(11) NOT NULL
-) ENGINE=MyISAM AUTO_INCREMENT=350435 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=413189 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -190,9 +190,9 @@ CREATE TABLE IF NOT EXISTS `build_logs_qa` (
 
 CREATE TABLE IF NOT EXISTS `build_logs_repoman` (
 `id` int(11) NOT NULL,
-  `build_logs_id` int(11) NOT NULL,
+  `build_log_id` int(11) NOT NULL,
   `summery_text` text NOT NULL
-) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=51 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -205,7 +205,7 @@ CREATE TABLE IF NOT EXISTS `build_logs_use` (
   `build_log_id` int(11) NOT NULL,
   `use_id` int(11) NOT NULL,
   `status` tinyint(1) NOT NULL
-) ENGINE=MyISAM AUTO_INCREMENT=66200 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=76204 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -281,7 +281,7 @@ CREATE TABLE IF NOT EXISTS `ebuilds` (
   `checksum` varchar(100) NOT NULL,
   `active` tinyint(1) NOT NULL DEFAULT '0',
   `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
-) ENGINE=MyISAM AUTO_INCREMENT=78536 DEFAULT CHARSET=utf8 COMMENT='Version main table (V)';
+) ENGINE=MyISAM AUTO_INCREMENT=81657 DEFAULT CHARSET=utf8 COMMENT='Version main table (V)';
 
 -- --------------------------------------------------------
 
@@ -294,7 +294,7 @@ CREATE TABLE IF NOT EXISTS `ebuilds_iuse` (
   `ebuild_id` int(11) NOT NULL,
   `use_id` int(11) NOT NULL,
   `status` tinyint(1) NOT NULL
-) ENGINE=MyISAM AUTO_INCREMENT=427416 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=456109 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -307,7 +307,7 @@ CREATE TABLE IF NOT EXISTS `ebuilds_keywords` (
   `ebuild_id` int(11) NOT NULL,
   `keyword_id` int(11) NOT NULL,
   `status` enum('Stable','Unstable','Negative') NOT NULL
-) ENGINE=MyISAM AUTO_INCREMENT=532825 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=559659 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -319,7 +319,7 @@ CREATE TABLE IF NOT EXISTS `ebuilds_metadata` (
 `id` int(11) NOT NULL,
   `ebuild_id` int(11) NOT NULL,
   `revision` varchar(10) NOT NULL COMMENT 'CVS revision'
-) ENGINE=MyISAM AUTO_INCREMENT=78536 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=81657 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -331,7 +331,7 @@ CREATE TABLE IF NOT EXISTS `ebuilds_restrictions` (
 `id` int(11) NOT NULL,
   `ebuild_id` int(11) NOT NULL,
   `restriction_id` int(11) NOT NULL
-) ENGINE=MyISAM AUTO_INCREMENT=13453 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=14328 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -342,7 +342,7 @@ CREATE TABLE IF NOT EXISTS `ebuilds_restrictions` (
 CREATE TABLE IF NOT EXISTS `emails` (
 `email_id` int(11) NOT NULL,
   `email` varchar(150) NOT NULL
-) ENGINE=MyISAM AUTO_INCREMENT=675 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=679 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -353,7 +353,7 @@ CREATE TABLE IF NOT EXISTS `emails` (
 CREATE TABLE IF NOT EXISTS `emerge_info` (
 `einfo_id` int(11) NOT NULL,
   `emerge_info_text` text NOT NULL
-) ENGINE=MyISAM AUTO_INCREMENT=16164 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=18635 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -445,7 +445,7 @@ CREATE TABLE IF NOT EXISTS `logs` (
   `log_type` enum('info','error','debug','qa') NOT NULL,
   `msg` text NOT NULL,
   `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
-) ENGINE=MyISAM AUTO_INCREMENT=9536253 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=22201403 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -461,7 +461,7 @@ CREATE TABLE IF NOT EXISTS `packages` (
   `checksum` varchar(100) NOT NULL,
   `active` tinyint(1) NOT NULL,
   `time_stamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
-) ENGINE=MyISAM AUTO_INCREMENT=18476 DEFAULT CHARSET=utf8 COMMENT='Packages main table (P)';
+) ENGINE=MyISAM AUTO_INCREMENT=18514 DEFAULT CHARSET=utf8 COMMENT='Packages main table (P)';
 
 -- --------------------------------------------------------
 
@@ -473,7 +473,7 @@ CREATE TABLE IF NOT EXISTS `packages_emails` (
 `id` int(11) NOT NULL,
   `package_id` int(11) NOT NULL,
   `email_id` int(11) NOT NULL
-) ENGINE=MyISAM AUTO_INCREMENT=22297 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=22365 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -485,7 +485,7 @@ CREATE TABLE IF NOT EXISTS `packages_metadata` (
 `id` int(11) NOT NULL,
   `package_id` int(11) NOT NULL,
   `checksum` varchar(100) NOT NULL
-) ENGINE=MyISAM AUTO_INCREMENT=18475 DEFAULT CHARSET=utf8;
+) ENGINE=MyISAM AUTO_INCREMENT=18513 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
 
@@ -517,7 +517,8 @@ CREATE TABLE IF NOT EXISTS `restrictions` (
 
 CREATE TABLE IF NOT EXISTS `setups` (
 `setup_id` int(11) NOT NULL,
-  `setup` varchar(100) NOT NULL
+  `setup` varchar(100) NOT NULL,
+  `profile` varchar(150) NOT NULL
 ) ENGINE=MyISAM AUTO_INCREMENT=4 DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
@@ -529,7 +530,7 @@ CREATE TABLE IF NOT EXISTS `setups` (
 CREATE TABLE IF NOT EXISTS `uses` (
 `use_id` int(11) NOT NULL,
   `flag` varchar(50) NOT NULL
-) ENGINE=MyISAM AUTO_INCREMENT=6153 DEFAULT CHARSET=utf8 COMMENT='Use flags main table';
+) ENGINE=MyISAM AUTO_INCREMENT=6180 DEFAULT CHARSET=utf8 COMMENT='Use flags main table';
 
 --
 -- Indexes for dumped tables
@@ -599,7 +600,7 @@ ALTER TABLE `build_logs_qa`
 -- Indexes for table `build_logs_repoman`
 --
 ALTER TABLE `build_logs_repoman`
- ADD PRIMARY KEY (`id`), ADD KEY `build_logs_id` (`build_logs_id`);
+ ADD PRIMARY KEY (`id`), ADD KEY `build_logs_id` (`build_log_id`);
 
 --
 -- Indexes for table `build_logs_use`
@@ -765,7 +766,7 @@ ALTER TABLE `uses`
 -- AUTO_INCREMENT for table `build_jobs`
 --
 ALTER TABLE `build_jobs`
-MODIFY `build_job_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=1325;
+MODIFY `build_job_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=3113;
 --
 -- AUTO_INCREMENT for table `build_jobs_emerge_options`
 --
@@ -780,17 +781,17 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
 -- AUTO_INCREMENT for table `build_jobs_use`
 --
 ALTER TABLE `build_jobs_use`
-MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=6067;
+MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=17343;
 --
 -- AUTO_INCREMENT for table `build_logs`
 --
 ALTER TABLE `build_logs`
-MODIFY `build_log_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=15154;
+MODIFY `build_log_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=17625;
 --
 -- AUTO_INCREMENT for table `build_logs_config`
 --
 ALTER TABLE `build_logs_config`
-MODIFY `log_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=41689;
+MODIFY `log_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=44160;
 --
 -- AUTO_INCREMENT for table `build_logs_emerge_options`
 --
@@ -800,12 +801,12 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
 -- AUTO_INCREMENT for table `build_logs_errors`
 --
 ALTER TABLE `build_logs_errors`
-MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=2616;
+MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=3379;
 --
 -- AUTO_INCREMENT for table `build_logs_hilight`
 --
 ALTER TABLE `build_logs_hilight`
-MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=350435;
+MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=413189;
 --
 -- AUTO_INCREMENT for table `build_logs_qa`
 --
@@ -815,12 +816,12 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
 -- AUTO_INCREMENT for table `build_logs_repoman`
 --
 ALTER TABLE `build_logs_repoman`
-MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=51;
 --
 -- AUTO_INCREMENT for table `build_logs_use`
 --
 ALTER TABLE `build_logs_use`
-MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=66200;
+MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=76204;
 --
 -- AUTO_INCREMENT for table `categories`
 --
@@ -845,37 +846,37 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=7;
 -- AUTO_INCREMENT for table `ebuilds`
 --
 ALTER TABLE `ebuilds`
-MODIFY `ebuild_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=78536;
+MODIFY `ebuild_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=81657;
 --
 -- AUTO_INCREMENT for table `ebuilds_iuse`
 --
 ALTER TABLE `ebuilds_iuse`
-MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=427416;
+MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=456109;
 --
 -- AUTO_INCREMENT for table `ebuilds_keywords`
 --
 ALTER TABLE `ebuilds_keywords`
-MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=532825;
+MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=559659;
 --
 -- AUTO_INCREMENT for table `ebuilds_metadata`
 --
 ALTER TABLE `ebuilds_metadata`
-MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=78536;
+MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=81657;
 --
 -- AUTO_INCREMENT for table `ebuilds_restrictions`
 --
 ALTER TABLE `ebuilds_restrictions`
-MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=13453;
+MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=14328;
 --
 -- AUTO_INCREMENT for table `emails`
 --
 ALTER TABLE `emails`
-MODIFY `email_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=675;
+MODIFY `email_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=679;
 --
 -- AUTO_INCREMENT for table `emerge_info`
 --
 ALTER TABLE `emerge_info`
-MODIFY `einfo_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=16164;
+MODIFY `einfo_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=18635;
 --
 -- AUTO_INCREMENT for table `emerge_options`
 --
@@ -910,22 +911,22 @@ MODIFY `keyword_id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'keyword index',AUTO
 -- AUTO_INCREMENT for table `logs`
 --
 ALTER TABLE `logs`
-MODIFY `log_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=9536253;
+MODIFY `log_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=22201403;
 --
 -- AUTO_INCREMENT for table `packages`
 --
 ALTER TABLE `packages`
-MODIFY `package_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=18476;
+MODIFY `package_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=18514;
 --
 -- AUTO_INCREMENT for table `packages_emails`
 --
 ALTER TABLE `packages_emails`
-MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=22297;
+MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=22365;
 --
 -- AUTO_INCREMENT for table `packages_metadata`
 --
 ALTER TABLE `packages_metadata`
-MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=18475;
+MODIFY `id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=18513;
 --
 -- AUTO_INCREMENT for table `repos`
 --
@@ -945,7 +946,7 @@ MODIFY `setup_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=4;
 -- AUTO_INCREMENT for table `uses`
 --
 ALTER TABLE `uses`
-MODIFY `use_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=6153;
+MODIFY `use_id` int(11) NOT NULL AUTO_INCREMENT,AUTO_INCREMENT=6180;
 DELIMITER $$
 --
 -- Events

diff --git a/tbc/pym/build_log.py b/tbc/pym/build_log.py
index a7d5314..bfbe8ec 100644
--- a/tbc/pym/build_log.py
+++ b/tbc/pym/build_log.py
@@ -23,16 +23,16 @@ portage.proxy.lazyimport.lazyimport(globals(),
 from tbc.repoman_tbc import tbc_repoman
 from tbc.text import get_log_text_dict
 from tbc.package import tbc_package
-from tbc.readconf import get_conf_settings
+from zobcs.readconf import read_config_settings
 from tbc.flags import tbc_use_flags
 from tbc.ConnectionManager import NewConnection
 from tbc.sqlquerys import add_tbc_logs, get_config_id, get_ebuild_id_db, add_new_buildlog, \
 	get_package_info, get_build_job_id, get_use_id, get_config_info, get_hilight_info, get_error_info_list, \
 	add_e_info, get_fail_times, add_fail_times, update_fail_times, del_old_build_jobs, add_old_ebuild, \
-	update_buildjobs_status
+	update_buildjobs_status, update_manifest_sql, add_repoman_qa
 from sqlalchemy.orm import sessionmaker
 
-def get_build_dict_db(session, config_id, settings, pkg):
+def get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg):
 	myportdb = portage.portdbapi(mysettings=settings)
 	cpvr_list = catpkgsplit(pkg.cpv, silent=1)
 	categories = cpvr_list[0]
@@ -77,10 +77,10 @@ def get_build_dict_db(session, config_id, settings, pkg):
 			log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
 			add_tbc_logs(session, log_msg, "info", config_id)
 			update_manifest_sql(session, build_dict['package_id'], "0")
-			init_package = tbc_package(session, settings, myportdb)
+			init_package = tbc_package(session, settings, myportdb, config_id, tbc_settings_dict)
 			init_package.update_package_db(build_dict['package_id'])
 			ebuild_id_list, status = get_ebuild_id_db(session, build_dict['checksum'], build_dict['package_id'])
-			if status and ebuild_id is None:
+			if status and ebuild_id_list is None:
 				log_msg = "%s:%s Don't have any ebuild_id!" % (pkg.cpv, repo,)
 				add_tbc_logs(session, log_msg, "error", config_id)
 		else:
@@ -209,8 +209,12 @@ def get_buildlog_info(session, settings, pkg, build_dict):
 	repoman_error_list = init_repoman.check_repoman(build_dict['cpv'], pkg.repo)
 	if repoman_error_list != []:
 		sum_build_log_list.append("1") # repoman = 1
+	else:
+		repoman_error_list = False
 	if qa_error_list != []:
 		sum_build_log_list.append("2") # qa = 2
+	else:
+		qa_error_list = False
 	error_search_line = "^ \\* ERROR: "
 	for error_log_line in error_log_list:
 		if re.search(error_search_line, error_log_line):
@@ -234,8 +238,7 @@ def get_emerge_info_id(settings, trees, session, config_id):
 	return "\n".join(emerge_info_list)
 
 def add_buildlog_main(settings, pkg, trees):
-	reader=get_conf_settings()
-	tbc_settings_dict=reader.read_tbc_settings_all()
+	tbc_settings_dict = read_config_settings()
 	config = tbc_settings_dict['tbc_config']
 	hostname =tbc_settings_dict['hostname']
 	host_config = hostname + "/" + config
@@ -245,7 +248,7 @@ def add_buildlog_main(settings, pkg, trees):
 	if pkg.type_name == "binary":
 		build_dict = None
 	else:
-		build_dict = get_build_dict_db(session, config_id, settings, pkg)
+		build_dict = get_build_dict_db(session, config_id, settings, tbc_settings_dict, pkg)
 	if build_dict is None:
 		log_msg = "Package %s:%s is NOT logged." % (pkg.cpv, pkg.repo,)
 		add_tbc_logs(session, log_msg, "info", config_id)
@@ -273,6 +276,7 @@ def add_buildlog_main(settings, pkg, trees):
 		log_msg = "Package %s:%s is NOT logged." % (pkg.cpv, pkg.repo,)
 		add_tbc_logs(session, log_msg, "info", config_id)
 	else:
+		add_repoman_qa(session, build_log_dict, log_id)
 		os.chmod(settings.get("PORTAGE_LOG_FILE"), 0o664)
 		log_msg = "Package: %s:%s is logged." % (pkg.cpv, pkg.repo,)
 		add_tbc_logs(session, log_msg, "info", config_id)
@@ -291,13 +295,9 @@ def log_fail_queru(session, build_dict, settings):
 	else:
 		build_log_dict = {}
 		error_log_list = []
-		qa_error_list = []
-		repoman_error_list = []
 		sum_build_log_list = []
 		sum_build_log_list.append("2")
 		error_log_list.append(build_dict['type_fail'])
-		build_log_dict['repoman_error_list'] = repoman_error_list
-		build_log_dict['qa_error_list'] = qa_error_list
 		build_log_dict['summary_error_list'] = sum_build_log_list
 		if build_dict['type_fail'] == 'merge fail':
 			error_log_list = []

diff --git a/tbc/pym/check_setup.py b/tbc/pym/check_setup.py
index 46d4391..afd6580 100644
--- a/tbc/pym/check_setup.py
+++ b/tbc/pym/check_setup.py
@@ -9,16 +9,19 @@ import errno
 from portage.exception import DigestException, FileNotFound, ParseError, PermissionDenied
 from tbc.text import get_file_text
 from tbc.sqlquerys import get_config_all_info, add_tbc_logs, get_configmetadata_info, get_setup_info
+from tbc.sync import git_pull
 
 def check_make_conf(session, config_id, tbc_settings_dict):
 	log_msg = "Checking configs for changes and errors"
 	add_tbc_logs(session, log_msg, "info", config_id)
+	git_repo = tbc_settings_dict['tbc_gitrepopath'] + "/"
+	git_pull(session, git_repo, config_id)
 	configsDict = {}
 	for ConfigInfo in get_config_all_info(session):
 		attDict={}
 		# Set the config dir
 		SetupInfo = get_setup_info(session, ConfigInfo.ConfigId)
-		check_config_dir = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + ConfigInfo.Hostname +"/" + SetupInfo.Setup + "/"
+		check_config_dir = tbc_settings_dict['tbc_gitrepopath'] + "/" + ConfigInfo.Hostname +"/" + SetupInfo.Setup + "/"
 		make_conf_file = check_config_dir + "etc/portage/make.conf"
 		ConfigsMetaDataInfo = get_configmetadata_info(session, ConfigInfo.ConfigId)
 		# Check if we can take a checksum on it.
@@ -48,7 +51,9 @@ def check_make_conf(session, config_id, tbc_settings_dict):
 	log_msg = "Checking configs for changes and errors ... Done"
 	add_tbc_logs(session, log_msg, "info", config_id)
 
-def check_make_conf_guest(session, config_id):
+def check_make_conf_guest(session, tbc_settings_dict, config_id):
+	git_repo = tbc_settings_dict['tbc_gitrepopath'] + "/"
+	git_pull(session, git_repo, config_id)
 	make_conf_file = "/etc/portage/make.conf"
 	# Check if we can open the file and close it
 	# Check if we have some error in the file (portage.util.getconfig)
@@ -68,7 +73,7 @@ def check_make_conf_guest(session, config_id):
 		return False
 	return True
 
-def check_configure_guest(session, config_id):
-	pass_make_conf = check_make_conf_guest(session, config_id)
+def check_configure_guest(session, tbc_settings_dict, config_id):
+	pass_make_conf = check_make_conf_guest(session, tbc_settings_dict, config_id)
 	print(pass_make_conf)
-	return pass_make_conf
\ No newline at end of file
+	return pass_make_conf

diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py
index c215456..31e1b66 100644
--- a/tbc/pym/db_mapping.py
+++ b/tbc/pym/db_mapping.py
@@ -16,6 +16,7 @@ class Keywords(Base):
 class Setups(Base):
 	SetupId = Column('setup_id', Integer, primary_key=True)
 	Setup = Column('setup', String(100))
+	Profile = Column('profile', String(150))
 	__tablename__ = 'setups'
 
 class Configs(Base):
@@ -46,7 +47,6 @@ class Jobs(Base):
 class ConfigsMetaData(Base):
 	Id = Column('id', Integer, primary_key=True)
 	ConfigId = Column('config_id', Integer, ForeignKey('configs.config_id'))
-	Profile = Column('profile', String(150))
 	KeywordId = Column('keyword_id', Integer, ForeignKey('keywords.keyword_id'))
 	MakeConfText = Column('make_conf_text', Text)
 	Checksum = Column('checksum', String(100))
@@ -216,6 +216,18 @@ class BuildLogsUse(Base):
 	Status = Column('status', Boolean, default=False)
 	__tablename__ = 'build_logs_use'
 
+class BuildLogsQA(Base):
+	Id = Column('id', Integer, primary_key=True)
+	BuildLogId = Column('build_log_id', Integer, ForeignKey('build_logs.build_log_id'))
+	SummeryText = Column('summery_text', Text)
+	__tablename__ = 'build_logs_qa'
+
+class BuildLogsRepoman(Base):
+	Id = Column('id', Integer, primary_key=True)
+	BuildLogId = Column('build_log_id', Integer, ForeignKey('build_logs.build_log_id'))
+	SummeryText = Column('summery_text', Text)
+	__tablename__ = 'build_logs_repoman'
+
 class ErrorsInfo(Base):
 	ErrorId = Column('error_id', Integer, primary_key=True)
 	ErrorName = Column('error_name', String)

diff --git a/tbc/pym/jobs.py b/tbc/pym/jobs.py
index 840932e..8559cf5 100644
--- a/tbc/pym/jobs.py
+++ b/tbc/pym/jobs.py
@@ -43,26 +43,11 @@ def jobs_main(session, config_id):
 			#	update_job_list(session, "Fail", job_id)
 			#	log_msg = "Job %s did fail." % (job_id,)
 			#	add_tbc_logs(session, log_msg, "info", config_id)
-		elif job == "gsync":
-			update_job_list(session, "Runing", job_id)
-			log_msg = "Job %s is runing." % (job_id,)
-			add_tbc_logs(session, log_msg, "info", config_id)
-			result = True
-			# result = git_pull(session)
-			if result:
-				update_job_list(session, "Done", job_id)
-				log_msg = "Job %s is done.." % (job_id,)
-				add_tbc_logs(session, log_msg, "info", config_id)
-			else:
-				update_job_list(session, "Fail", job_id)
-				log_msg = "Job %s did fail." % (job_id,)
-				add_tbc_logs(session, log_msg, "info", config_id)
 		elif job == "esync":
 			update_job_list(session, "Runing", job_id)
 			log_msg = "Job %s is runing." % (job_id,)
 			add_tbc_logs(session, log_msg, "info", config_id)
-			result =  sync_tree(session)
-			if result:
+			if sync_tree(session) and update_db_main(session, config_id):
 				update_job_list(session, "Done", job_id)
 				log_msg = "Job %s is done.." % (job_id,)
 				add_tbc_logs(session, log_msg, "info", config_id)
@@ -74,8 +59,7 @@ def jobs_main(session, config_id):
 			update_job_list(session, "Runing", job_id)
 			log_msg = "Job %s is runing." % (job_id,)
 			add_tbc_logs(session, log_msg, "info", config_id)
-			result = update_db_main(session, config_id)
-			if result:
+			ifupdate_db_main(session, config_id):
 				update_job_list(session, "Done", job_id)
 				log_msg = "Job %s is done.." % (job_id,)
 				add_tbc_logs(session, log_msg, "info", config_id)

diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index af1bff5..5f172cb 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -26,7 +26,7 @@ class tbc_package(object):
 
 	def change_config(self, host_config):
 		# Change config_root  config_setup = table config
-		my_new_setup = "/var/cache/tbc/" + self._tbc_settings_dict['tbc_gitreponame'] + "/" + host_config + "/"
+		my_new_setup = tbc_settings_dict['tbc_gitrepopath'] + "/" + host_config + "/"
 		mysettings_setup = portage.config(config_root = my_new_setup)
 		return mysettings_setup
 

diff --git a/tbc/pym/readconf.py b/tbc/pym/readconf.py
index db1056e..39131db 100644
--- a/tbc/pym/readconf.py
+++ b/tbc/pym/readconf.py
@@ -32,10 +32,10 @@ class get_conf_settings(object):
 			if element[0] == 'SQLPASSWD':		# Password
 				get_sql_passwd = element[1]
 			# Buildhost root (dir for host/setup on host)
-			if element[0] == 'ZOBCSGITREPONAME':
-				get_tbc_gitreponame = element[1]
+			if element[0] == 'TBCREPOPATH':
+				get_tbc_gitrepopath = element[1]
 			# Buildhost setup (host/setup on guest)
-			if element[0] == 'ZOBCSCONFIG':
+			if element[0] == 'TBCCONFIG':
 				get_tbc_config = element[1]
 			# if element[0] == 'LOGFILE':
 			#	get_tbc_logfile = element[1]
@@ -47,7 +47,7 @@ class get_conf_settings(object):
 		tbc_settings_dict['sql_host'] = get_sql_host.rstrip('\n')
 		tbc_settings_dict['sql_user'] = get_sql_user.rstrip('\n')
 		tbc_settings_dict['sql_passwd'] = get_sql_passwd.rstrip('\n')
-		tbc_settings_dict['tbc_gitreponame'] = get_tbc_gitreponame.rstrip('\n')
+		tbc_settings_dict['tbc_gitrepopath'] = get_tbc_gitrepopath.rstrip('\n')
 		tbc_settings_dict['tbc_config'] = get_tbc_config.rstrip('\n')
 		tbc_settings_dict['hostname'] = getfqdn()
 		# tbc_settings_dict['tbc_logfile'] = get_tbc_logfile.rstrip('\n')

diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py
index 623f0b0..f9fc8b4 100644
--- a/tbc/pym/sqlquerys.py
+++ b/tbc/pym/sqlquerys.py
@@ -7,7 +7,7 @@ from tbc.db_mapping import Configs, Logs, ConfigsMetaData, Jobs, BuildJobs, Pack
 	Uses, ConfigsEmergeOptions, EmergeOptions, HiLight, BuildLogs, BuildLogsConfig, BuildJobsUse, BuildJobsRedo, \
 	HiLightCss, BuildLogsHiLight, BuildLogsEmergeOptions, BuildLogsErrors, ErrorsInfo, EmergeInfo, BuildLogsUse, \
 	BuildJobsEmergeOptions, EbuildsMetadata, EbuildsIUse, Restrictions, EbuildsRestrictions, EbuildsKeywords, \
-        Keywords, PackagesMetadata, Emails, PackagesEmails, Setups
+        Keywords, PackagesMetadata, Emails, PackagesEmails, Setups, BuildLogsQA,  BuildLogsRepoman
 from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 from sqlalchemy import and_, or_
 
@@ -69,15 +69,17 @@ def is_build_job_done(session, build_job_id):
 def get_packages_to_build(session, config_id):
 	SetupInfo = get_setup_info(session, config_id)
 	BuildJobsTmp = session.query(BuildJobs).filter(BuildJobs.SetupId==SetupInfo.SetupId). \
-				order_by(BuildJobs.BuildJobId)
-	if session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(BuildNow = True).filter_by(Status = 'Waiting').all() == [] and session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Status = 'Waiting').all() == []:
+		order_by(BuildJobs.BuildJobId).filter_by(Status = 'Waiting')
+	if BuildJobsTmp.all() == []:
 		return None
-	if not BuildJobsTmp.filter_by(BuildNow = True).first() is None:
-		BuildJobsInfo = session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(BuildNow = True). \
-			filter_by(Status = 'Waiting').order_by(BuildJobs.BuildJobId).first()
+	elif BuildJobsTmp.filter_by(BuildNow = True).all() != []:
+		BuildJobsInfo = BuildJobsTmp.filter_by(BuildNow = True).first()
+	elif BuildJobsTmp.filter_by(BuildNow = False).all() != []:
+		BuildJobsInfo = BuildJobsTmp.filter_by(BuildNow = False).first()
 	else:
-		BuildJobsInfo = session.query(BuildJobs).filter_by(SetupId = SetupInfo.SetupId).filter_by(Status = 'Waiting').\
-			order_by(BuildJobs.BuildJobId).first()
+		log_msg = "BuildJobsTmp found job but the if state mant did not."
+		add_zobcs_logs(session, log_msg, "error", config_id)
+		return None
 	update_buildjobs_status(session, BuildJobsInfo.BuildJobId, 'Looked', config_id)
 	EbuildsInfo = session.query(Ebuilds).filter_by(EbuildId = BuildJobsInfo.EbuildId).one()
 	PackagesInfo, CategoriesInfo = session.query(Packages, Categories).filter(Packages.PackageId==EbuildsInfo.PackageId).filter(Packages.CategoryId==Categories.CategoryId).one()
@@ -266,6 +268,22 @@ def add_new_buildlog(session, build_dict, build_log_dict):
 		del_old_build_jobs(session, build_dict['build_job_id'])
 		return build_log_id
 
+def add_repoman_qa(session, build_log_dict, log_id):
+	repoman_error = ""
+	qa_error = ""
+	if build_log_dict['qa_error_list']:
+		for qa_text in build_log_dict['qa_error_list']:
+			qa_error = qa_error + build_log_dict['qa_error_list']
+		NewBuildLogQA = BuildLogsQA(BuildLogId = log_id, SummeryText = qa_error)
+		session.add(NewBuildLogQA)
+		session.commit()
+	if build_log_dict['repoman_error_list']:
+		for repoman_text in build_log_dict['repoman_error_list']:
+			repoman_error = repoman_error + repoman_text
+		NewBuildLogRepoman = BuildLogsRepoman(BuildLogId = log_id, SummeryText = repoman_error)
+		session.add(NewBuildLogRepoman)
+		session.commit()
+
 def update_fail_times(session, FailInfo):
 	NewBuildJobs = session.query(BuildJobs).filter_by(BuildJobId = FailInfo.BuildJobId).one()
 	NewBuildJobs.TimeStamp = datetime.datetime.utcnow()
@@ -279,7 +297,6 @@ def get_fail_times(session, build_dict):
 	return True
 
 def add_fail_times(session, fail_querue_dict):
-	print(fail_querue_dict)
 	NewBuildJobsRedo = BuildJobsRedo(BuildJobId = fail_querue_dict['build_job_id'], FailType = fail_querue_dict['fail_type'], FailTimes = fail_querue_dict['fail_times'])
 	session.add(NewBuildJobsRedo)
 	session.commit()

diff --git a/tbc/pym/sync.py b/tbc/pym/sync.py
index a75562f..f1bc56a 100644
--- a/tbc/pym/sync.py
+++ b/tbc/pym/sync.py
@@ -7,9 +7,10 @@ import os
 import errno
 import sys
 import time
+from pygit2 import Repository, GIT_MERGE_ANALYSIS_FASTFORWARD, GIT_MERGE_ANALYSIS_NORMAL, \
+        GIT_MERGE_ANALYSIS_UP_TO_DATE
 
 from _emerge.main import emerge_main
-from tbc.readconf import get_conf_settings
 from tbc.sqlquerys import get_config_id, add_tbc_logs, get_config_all_info, get_configmetadata_info
 from tbc.updatedb import update_db_main
 from tbc.readconf import read_config_settings
@@ -20,7 +21,7 @@ def sync_tree(session):
 	_config = tbc_settings_dict['tbc_config']
 	config_id = get_config_id(session, _config, _hostname)
 	host_config = _hostname +"/" + _config
-	default_config_root = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + host_config + "/"
+	default_config_root = tbc_settings_dict['tbc_gitrepopath']  + "/" + host_config + "/"
 	mysettings = portage.config(config_root = default_config_root)
 	GuestBusy = True
 	log_msg = "Waiting for Guest to be idel"
@@ -65,9 +66,38 @@ def sync_tree(session):
 			pass
 		log_msg = "Emerge --sync ... Done."
 		add_tbc_logs(session, log_msg, "info", config_id)
-	result = update_db_main(session, config_id)
-	if result:
-		return True
+	return True
+
+def git_pull(session, git_repo, config_id):
+	log_msg = "Git pull"
+	add_zobcs_logs(session, log_msg, "info", config_id)
+	repo = Repository(git_repo + ".git")
+	remote = repo.remotes["origin"]
+	remote.fetch()
+	remote_master_id = repo.lookup_reference('refs/remotes/origin/master').target
+	merge_result, _ = repo.merge_analysis(remote_master_id)
+	if merge_result & GIT_MERGE_ANALYSIS_UP_TO_DATE:
+		log_msg = "Repo is up to date"
+		add_zobcs_logs(session, log_msg, "info", config_id)
+	elif merge_result & GIT_MERGE_ANALYSIS_FASTFORWARD:
+		repo.checkout_tree(repo.get(remote_master_id))
+		master_ref = repo.lookup_reference('refs/heads/master')
+		master_ref.set_target(remote_master_id)
+		repo.head.set_target(remote_master_id)
+	elif merge_result & GIT_MERGE_ANALYSIS_NORMAL:
+		repo.merge(remote_master_id)
+		assert repo.index.conflicts is None, 'Conflicts, ahhhh!'
+		user = repo.default_signature
+		tree = repo.index.write_tree()
+		commit = repo.create_commit('HEAD',
+			user,
+			user,
+			'Merge!',
+			tree,
+			[repo.head.target, remote_master_id])
+		repo.state_cleanup()
 	else:
-		log_msg = "Updatedb fail"
-		add_tbc_logs(session, log_msg, "info", config_id)
+		raise AssertionError('Unknown merge analysis result')
+	log_msg = "Git pull ... Done"
+	add_zobcs_logs(session, log_msg, "info", config_id)
+	return True

diff --git a/tbc/pym/updatedb.py b/tbc/pym/updatedb.py
index 12b94c5..90a9e0a 100644
--- a/tbc/pym/updatedb.py
+++ b/tbc/pym/updatedb.py
@@ -25,7 +25,7 @@ def init_portage_settings(session, config_id, tbc_settings_dict):
 	
 	# Get default config from the configs table  and default_config=1
 	host_config = tbc_settings_dict['hostname'] +"/" + tbc_settings_dict['tbc_config']
-	default_config_root = "/var/cache/tbc/" + tbc_settings_dict['tbc_gitreponame'] + "/" + host_config + "/"
+	default_config_root = tbc_settings_dict['tbc_gitrepopath'] + "/" + host_config + "/"
 
 	# Set config_root (PORTAGE_CONFIGROOT)  to default_config_root
 	mysettings = portage.config(config_root = default_config_root)


^ permalink raw reply related	[flat|nested] 2+ messages in thread
* [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/, sql/
@ 2015-07-21 15:48 Magnus Granberg
  0 siblings, 0 replies; 2+ messages in thread
From: Magnus Granberg @ 2015-07-21 15:48 UTC (permalink / raw
  To: gentoo-commits

commit:     2bd236b7fefd255aee66d70848906c66f5da056a
Author:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
AuthorDate: Tue Jul 21 15:48:22 2015 +0000
Commit:     Magnus Granberg <zorry <AT> gentoo <DOT> org>
CommitDate: Tue Jul 21 15:48:22 2015 +0000
URL:        https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=2bd236b7

add support for changlog

 sql/structure_dump.sql | 32 +++++++++++++++++++++++++++++---
 tbc/pym/db_mapping.py  |  2 ++
 tbc/pym/package.py     | 34 +++++++++++++++++++++++++++-------
 tbc/pym/sqlquerys.py   |  4 +++-
 4 files changed, 61 insertions(+), 11 deletions(-)

diff --git a/sql/structure_dump.sql b/sql/structure_dump.sql
index 459f6bf..cb1b30f 100644
--- a/sql/structure_dump.sql
+++ b/sql/structure_dump.sql
@@ -3,7 +3,7 @@
 -- http://www.phpmyadmin.net
 --
 -- Host: localhost
--- Generation Time: Jul 17, 2015 at 08:35 PM
+-- Generation Time: Jul 21, 2015 at 03:29 PM
 -- Server version: 10.0.15-MariaDB-log
 -- PHP Version: 5.6.10-pl0-gentoo
 
@@ -213,6 +213,19 @@ CREATE TABLE IF NOT EXISTS `categories` (
 -- --------------------------------------------------------
 
 --
+-- Table structure for table `categories_metadata`
+--
+
+CREATE TABLE IF NOT EXISTS `categories_metadata` (
+`id` int(11) NOT NULL,
+  `category_id` int(11) NOT NULL,
+  `checksum` varchar(100) NOT NULL,
+  `descriptions` text NOT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COMMENT='Categories main table (C)';
+
+-- --------------------------------------------------------
+
+--
 -- Table structure for table `configs`
 --
 
@@ -474,7 +487,9 @@ CREATE TABLE IF NOT EXISTS `packages_emails` (
 CREATE TABLE IF NOT EXISTS `packages_metadata` (
 `id` int(11) NOT NULL,
   `package_id` int(11) NOT NULL,
-  `checksum` varchar(100) NOT NULL
+  `checksum` varchar(100) NOT NULL,
+  `checksum_chlog` varchar(100) NOT NULL,
+  `changlog` text NOT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
@@ -599,6 +614,12 @@ ALTER TABLE `categories`
  ADD PRIMARY KEY (`category_id`);
 
 --
+-- Indexes for table `categories_metadata`
+--
+ALTER TABLE `categories_metadata`
+ ADD PRIMARY KEY (`id`);
+
+--
 -- Indexes for table `configs`
 --
 ALTER TABLE `configs`
@@ -807,6 +828,11 @@ MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
 ALTER TABLE `categories`
 MODIFY `category_id` int(11) NOT NULL AUTO_INCREMENT;
 --
+-- AUTO_INCREMENT for table `categories_metadata`
+--
+ALTER TABLE `categories_metadata`
+MODIFY `id` int(11) NOT NULL AUTO_INCREMENT;
+--
 -- AUTO_INCREMENT for table `configs`
 --
 ALTER TABLE `configs`
@@ -930,7 +956,7 @@ DELIMITER $$
 --
 -- Events
 --
-CREATE DEFINER=`tbc`@`localhost` EVENT `add_esync_jobs` ON SCHEDULE EVERY 1 HOUR STARTS '2012-12-23 17:15:13' ON COMPLETION NOT PRESERVE ENABLE DO BEGIN
+CREATE DEFINER=`tbc`@`localhost` EVENT `add_esync_jobs` ON SCHEDULE EVERY 30 MINUTE STARTS '2012-12-23 17:15:13' ON COMPLETION NOT PRESERVE ENABLE DO BEGIN
   CALL add_jobs_esync();
 END$$
 

diff --git a/tbc/pym/db_mapping.py b/tbc/pym/db_mapping.py
index cf4a054..37eaec8 100644
--- a/tbc/pym/db_mapping.py
+++ b/tbc/pym/db_mapping.py
@@ -104,6 +104,8 @@ class PackagesMetadata(Base):
 	Id = Column('id', Integer, primary_key=True)
 	PackageId = Column('package_id', Integer, ForeignKey('packages.package_id'))
 	Checksum = Column('checksum', String(100))
+	ChecksumChanglog = Column('checksum_chlog', String(100))
+	Changlog = Column('changlog', Text)
 	__tablename__ = 'packages_metadata'
 
 class Ebuilds(Base):

diff --git a/tbc/pym/package.py b/tbc/pym/package.py
index 8d156d8..c8a70db 100644
--- a/tbc/pym/package.py
+++ b/tbc/pym/package.py
@@ -2,10 +2,11 @@
 # Distributed under the terms of the GNU General Public License v2
 
 from __future__ import print_function
+import re
 import portage
 from portage.xml.metadata import MetaDataXML
 from tbc.flags import tbc_use_flags
-from tbc.text import get_ebuild_cvs_revision
+from tbc.text import get_ebuild_cvs_revision, get_log_text_dict
 from tbc.flags import tbc_use_flags
 from tbc.qachecks import digestcheck, check_repoman
 from tbc.sqlquerys import add_logs, get_package_info, get_config_info, \
@@ -156,19 +157,37 @@ class tbc_package(object):
 						log_msg = "B %s:%s USE: %s Setup: %s" % (k, v['repo'], use_flagsDict, setup_id,)
 						add_logs(self._session, log_msg, "info", self._config_id)
 					i = i +1
+	def get_changelog_text(self,pkgdir):
+		changelog_text_dict, max_text_lines = get_log_text_dict(pkgdir + "/ChangeLog")
+		spec = 3
+		spec_tmp = 1
+		changelog_text_tree = ''
+		for index, text_line inchangelog_text_dict.items():
+			if re.search('^#', text_line)
+				changelog_text_tree = changelog_text_tree + text_line
+			elif re.search('^\n', text_line) and re.search('^\*', changelog_text_dict[index + 1])
+				changelog_text_tree = changelog_text_tree + text_line
+				spec_tmp = spec_tmp + 1
+				spec = spec + 1
+			elif re.search('^\n', text_line) and not re.search('^\*', changelog_text_dict[index + 1])
+				if spec_tmp == spec:
+					break
+				else
+					spec_tmp = spec_tmp + 1
+					changelog_text_tree = changelog_text_tree + text_line
+			else:
+				changelog_text_tree = changelog_text_tree + text_line
+		return changelog_text_tree
 
 	def get_package_metadataDict(self, pkgdir, package_id):
 		# Make package_metadataDict
 		attDict = {}
 		package_metadataDict = {}
 		md_email_list = []
-		# changelog_checksum_tree = portage.checksum.sha256hash(pkgdir + "/ChangeLog")
-		# changelog_text_tree = get_file_text(pkgdir + "/ChangeLog")
 		herd = None
 		pkg_md = MetaDataXML(pkgdir + "/metadata.xml", herd)
-		#metadata_xml_text_tree = get_file_text(pkgdir + "/metadata.xml")
-		# attDict['changelog_checksum'] =  changelog_checksum_tree[0]
-		# attDict['changelog_text'] =  changelog_text_tree
+		attDict['changelog_checksum'] =   portage.checksum.sha256hash(pkgdir + "/ChangeLog")[0]
+		attDict['changelog_text'] =  self.get_changelog_text(pkgdir)
 		tmp_herds = pkg_md.herds()
 		if tmp_herds != ():
 			attDict['metadata_xml_herds'] = tmp_herds[0]
@@ -181,7 +200,8 @@ class tbc_package(object):
 			log_msg = "Metadata file %s missing Email" % (pkgdir + "/metadata.xml")
 			add_logs(self._session, log_msg, "qa", self._config_id)
 			attDict['metadata_xml_email'] = False
-		attDict['metadata_xml_descriptions'] = pkg_md.descriptions()
+		metadata_xml_descriptions_tree = re.sub('\t', '', pkg_md.descriptions()[0])
+        attDict['metadata_xml_descriptions'] = re.sub('\n', '', metadata_xml_descriptions_tree)
 		attDict['metadata_xml_checksum'] =  portage.checksum.sha256hash(pkgdir + "/metadata.xml")[0]
 		#attDict['metadata_xml_text'] =  metadata_xml_text_tree
 		package_metadataDict[package_id] = attDict

diff --git a/tbc/pym/sqlquerys.py b/tbc/pym/sqlquerys.py
index a521dbc..7a64db7 100644
--- a/tbc/pym/sqlquerys.py
+++ b/tbc/pym/sqlquerys.py
@@ -490,10 +490,12 @@ def update_package_metadata(session, package_metadataDict):
 		try:
 			PackagesMetadataInfo = session.query(PackagesMetadata).filter_by(PackageId = k).one()
 		except NoResultFound as e:
-			session.add(PackagesMetadata(PackageId = k, Checksum = v['metadata_xml_checksum']))
+			session.add(PackagesMetadata(PackageId = k, Checksum = v['metadata_xml_checksum'], ChecksumChanglog = v['changelog_checksum'], Changlog = v['changelog_text']))
 			session.commit()
 		else:
 			PackagesMetadataInfo.Checksum = v['metadata_xml_checksum']
+			PackagesMetadataInfo.ChecksumChanglog = v['changelog_checksum']
+			PackagesMetadataInfo.Changlog = v['changelog_text']
 			session.commit()
 		if v['metadata_xml_email']:
 			for email in v['metadata_xml_email']:


^ permalink raw reply related	[flat|nested] 2+ messages in thread
end of thread, other threads:[~2015-07-21 15:49 UTC | newest]

Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2015-07-21 15:48 [gentoo-commits] proj/tinderbox-cluster:master commit in: tbc/pym/, sql/ Magnus Granberg
  -- strict thread matches above, loose matches on Subject: below --
2015-05-09 15:03 Magnus Granberg

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox