* [gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/, catalyst/targets/
2015-01-01 5:59 [gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/base/, catalyst/targets/ Brian Dolbec
@ 2015-02-26 20:12 ` Brian Dolbec
0 siblings, 0 replies; 4+ messages in thread
From: Brian Dolbec @ 2015-02-26 20:12 UTC (permalink / raw
To: gentoo-commits
commit: f98b415f062b624bde02d48768372c9848abe175
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Fri May 31 16:11:30 2013 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Thu Jan 1 05:58:07 2015 +0000
URL: http://sources.gentoo.org/gitweb/?p=proj/catalyst.git;a=commit;h=f98b415f
Initial creation of fileops.py
Migrate to using ensure_dirs()
---
catalyst/base/clearbase.py | 10 ++++-----
catalyst/base/stagebase.py | 32 +++++++++++-----------------
catalyst/fileops.py | 45 +++++++++++++++++++++++++++++++++++++++
catalyst/targets/grp.py | 5 ++---
catalyst/targets/livecd_stage1.py | 4 ++--
catalyst/targets/livecd_stage2.py | 13 ++++-------
catalyst/targets/netboot2.py | 6 +++---
catalyst/targets/snapshot.py | 10 ++++-----
catalyst/targets/stage1.py | 10 ++++-----
9 files changed, 81 insertions(+), 54 deletions(-)
diff --git a/catalyst/base/clearbase.py b/catalyst/base/clearbase.py
index 8519acc..585d4f2 100644
--- a/catalyst/base/clearbase.py
+++ b/catalyst/base/clearbase.py
@@ -5,7 +5,7 @@ from stat import ST_UID, ST_GID, ST_MODE
from catalyst.support import cmd, countdown
-
+from catalyst.fileops import ensure_dirs
class ClearBase(object):
"""
@@ -35,7 +35,7 @@ class ClearBase(object):
+myemp)
#cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env-self.env)
shutil.rmtree(myemp)
- os.makedirs(myemp,0755)
+ ensure_dirs(myemp, 0755)
os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
os.chmod(myemp,mystat[ST_MODE])
@@ -54,7 +54,7 @@ class ClearBase(object):
if os.uname()[0] == "FreeBSD":
os.system("chflags -R noschg "+myemp)
shutil.rmtree(myemp)
- os.makedirs(myemp,0755)
+ ensure_dirs(myemp, mode=0755)
os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
os.chmod(myemp,mystat[ST_MODE])
@@ -73,7 +73,7 @@ class ClearBase(object):
mystat=os.stat(myemp)
#cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
shutil.rmtree(myemp)
- os.makedirs(myemp,0755)
+ ensure_dirs(myemp, mode=0755)
os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
os.chmod(myemp,mystat[ST_MODE])
@@ -92,7 +92,7 @@ class ClearBase(object):
mystat=os.stat(myemp)
#cmd("rm -rf "+myemp, "Could not remove existing file: "+myemp,env=self.env)
shutil.rmtree(myemp)
- os.makedirs(myemp,0755)
+ ensure_dirs(myemp, mode=0755)
os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
os.chmod(myemp,mystat[ST_MODE])
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index 3056ce2..40c7362 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -18,6 +18,7 @@ from catalyst.base.targetbase import TargetBase
from catalyst.base.clearbase import ClearBase
from catalyst.base.genbase import GenBase
from catalyst.lock import LockDir
+from catalyst.fileops import ensure_dirs, pjoin
class StageBase(TargetBase, ClearBase, GenBase):
@@ -339,8 +340,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
# +self.settings["target_path"],env=self.env)
touch(setup_target_path_resume)
- if not os.path.exists(self.settings["storedir"]+"/builds/"):
- os.makedirs(self.settings["storedir"]+"/builds/")
+ ensure_dirs(self.settings["storedir"] + "/builds/")
def set_fsscript(self):
if self.settings["spec_prefix"]+"/fsscript" in self.settings:
@@ -486,8 +486,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
))
if "autoresume" in self.settings["options"]:
print "The autoresume path is " + self.settings["autoresume_path"]
- if not os.path.exists(self.settings["autoresume_path"]):
- os.makedirs(self.settings["autoresume_path"],0755)
+ ensure_dirs(self.settings["autoresume_path"], mode=0755)
def set_controller_file(self):
self.settings["controller_file"]=normpath(self.settings["sharedir"]+\
@@ -769,19 +768,15 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.clear_autoresume()
self.clear_chroot()
- if not os.path.exists(self.settings["chroot_path"]):
- os.makedirs(self.settings["chroot_path"])
+ ensure_dirs(self.settings["chroot_path"])
- if not os.path.exists(self.settings["chroot_path"]+"/tmp"):
- os.makedirs(self.settings["chroot_path"]+"/tmp",1777)
+ ensure_dirs(self.settings["chroot_path"]+"/tmp",mode=1777)
if "pkgcache" in self.settings["options"]:
- if not os.path.exists(self.settings["pkgcache_path"]):
- os.makedirs(self.settings["pkgcache_path"],0755)
+ ensure_dirs(self.settings["pkgcache_path"],mode=0755)
if "kerncache" in self.settings["options"]:
- if not os.path.exists(self.settings["kerncache_path"]):
- os.makedirs(self.settings["kerncache_path"],0755)
+ ensure_dirs(self.settings["kerncache_path"],mode=0755)
print display_msg
cmd(unpack_cmd,error_msg,env=self.env)
@@ -849,8 +844,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
print cleanup_msg
cleanup_cmd="rm -rf "+destdir
cmd(cleanup_cmd,cleanup_errmsg,env=self.env)
- if not os.path.exists(destdir):
- os.makedirs(destdir,0755)
+ ensure_dirs(destdir,mode=0755)
print "Unpacking portage tree (This can take a long time) ..."
cmd(unpack_cmd,unpack_errmsg,env=self.env)
@@ -935,12 +929,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
for x in self.mounts:
#print "bind(); x =", x
target = normpath(self.settings["chroot_path"] + self.target_mounts[x])
- if not os.path.exists(target):
- os.makedirs(target, 0755)
+ ensure_dirs(target, mode=0755)
if not os.path.exists(self.mountmap[x]):
if self.mountmap[x] not in ["tmpfs", "shmfs"]:
- os.makedirs(self.mountmap[x], 0755)
+ ensure_dirs(self.mountmap[x], mode=0755)
src=self.mountmap[x]
#print "bind(); src =", src
@@ -1210,7 +1203,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
"""
mystat=os.stat(myemp)
shutil.rmtree(myemp)
- os.makedirs(myemp,0755)
+ ensure_dirs(myemp, mode=0755)
os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
os.chmod(myemp,mystat[ST_MODE])
touch(empty_resume)
@@ -1266,8 +1259,7 @@ class StageBase(TargetBase, ClearBase, GenBase):
mypath=string.join(mypath[:-1],"/")
""" Now make sure path exists """
- if not os.path.exists(mypath):
- os.makedirs(mypath)
+ ensure_dirs(mypath)
print "Creating stage tarball..."
diff --git a/catalyst/fileops.py b/catalyst/fileops.py
new file mode 100644
index 0000000..009bf1c
--- /dev/null
+++ b/catalyst/fileops.py
@@ -0,0 +1,45 @@
+#!/usr/bin/python
+
+# Maintained in full by:
+# Catalyst Team <catalyst@gentoo.org>
+# Release Engineering Team <releng@gentoo.org>
+
+'''fileops.py
+
+Performs file operations such as pack/unpack,
+ensuring directories exist,... imports snakeoils osutils
+functions for use throughout catalyst.
+'''
+
+from snakeoil.osutils import (ensure_dirs as snakeoil_ensure_dirs, normpath,
+ pjoin, listdir_files)
+from catalyst.support import CatalystError
+
+
+def ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True,
+ failback=None, fatal=False):
+ '''Wrapper to snakeoil.osutil's ensure_dirs()
+ This additionally allows for failures to run
+ cleanup or other code and/or raise fatal errors.
+
+ :param path: directory to ensure exists on disk
+ :param gid: a valid GID to set any created directories to
+ :param uid: a valid UID to set any created directories to
+ :param mode: permissions to set any created directories to
+ :param minimal: boolean controlling whether or not the specified mode
+ must be enforced, or is the minimal permissions necessary. For example,
+ if mode=0755, minimal=True, and a directory exists with mode 0707,
+ this will restore the missing group perms resulting in 757.
+ :param failback: function to run in the event of a failed attemp
+ to create the directory.
+ :return: True if the directory could be created/ensured to have those
+ permissions, False if not.
+ '''
+ succeeded = snakeoil_ensure_dirs(path, gid=-1, uid=-1, mode=0777, minimal=True)
+ if not succeeded:
+ if failback:
+ failback()
+ if fatal:
+ raise CatalystError(
+ "Failed to create directory: %s" % path, print_traceback=True)
+ return succeeded
diff --git a/catalyst/targets/grp.py b/catalyst/targets/grp.py
index 93d30fc..a17f9bd 100644
--- a/catalyst/targets/grp.py
+++ b/catalyst/targets/grp.py
@@ -10,7 +10,7 @@ import glob
from catalyst.support import (CatalystError, normpath,
touch, cmd, list_bashify)
-
+from catalyst.fileops import ensure_dirs
from catalyst.base.stagebase import StageBase
@@ -51,8 +51,7 @@ class grp(StageBase):
#if os.path.isdir(self.settings["target_path"]):
#cmd("rm -rf "+self.settings["target_path"],
#"Could not remove existing directory: "+self.settings["target_path"],env=self.env)
- if not os.path.exists(self.settings["target_path"]):
- os.makedirs(self.settings["target_path"])
+ ensure_dirs(self.settings["target_path"])
touch(self.settings["autoresume_path"]+"setup_target_path")
diff --git a/catalyst/targets/livecd_stage1.py b/catalyst/targets/livecd_stage1.py
index 3b19c3f..668960b 100644
--- a/catalyst/targets/livecd_stage1.py
+++ b/catalyst/targets/livecd_stage1.py
@@ -10,6 +10,7 @@ import string
from catalyst.support import (normpath,
touch, cmd)
+from catalyst.fileops import ensure_dirs
from catalyst.base.stagebase import StageBase
@@ -43,8 +44,7 @@ class livecd_stage1(StageBase):
"Could not remove existing directory: "+self.settings["target_path"],env=self.env)
touch(self.settings["autoresume_path"]+"setup_target_path")
- if not os.path.exists(self.settings["target_path"]):
- os.makedirs(self.settings["target_path"])
+ ensure_dirs(self.settings["target_path"])
def set_spec_prefix(self):
self.settings["spec_prefix"]="livecd"
diff --git a/catalyst/targets/livecd_stage2.py b/catalyst/targets/livecd_stage2.py
index b1cf6b2..94da670 100644
--- a/catalyst/targets/livecd_stage2.py
+++ b/catalyst/targets/livecd_stage2.py
@@ -7,6 +7,7 @@ import os
from catalyst.support import (normpath, file_locate, CatalystError, cmd,
read_from_clst, touch)
+from catalyst.fileops import ensure_dirs
from catalyst.base.stagebase import StageBase
@@ -62,8 +63,7 @@ class livecd_stage2(StageBase):
cmd("rm -rf "+self.settings["target_path"],
"Could not remove existing directory: "+self.settings["target_path"],env=self.env)
touch(self.settings["autoresume_path"]+"setup_target_path")
- if not os.path.exists(self.settings["target_path"]):
- os.makedirs(self.settings["target_path"])
+ ensure_dirs(self.settings["target_path"])
def run_local(self):
# what modules do we want to blacklist?
@@ -116,15 +116,10 @@ class livecd_stage2(StageBase):
self.clear_chroot()
#self.dir_setup()
- if not os.path.exists(self.settings["chroot_path"]):
- os.makedirs(self.settings["chroot_path"])
-
- if not os.path.exists(self.settings["chroot_path"]+"/tmp"):
- os.makedirs(self.settings["chroot_path"]+"/tmp",1777)
+ ensure_dirs(self.settings["chroot_path"]+"/tmp", mode=1777)
if "pkgcache" in self.settings["options"]:
- if not os.path.exists(self.settings["pkgcache_path"]):
- os.makedirs(self.settings["pkgcache_path"],0755)
+ ensure_dirs(self.settings["pkgcache_path"], mode=0755)
if not display_msg:
raise CatalystError("Could not find appropriate source.\n"
diff --git a/catalyst/targets/netboot2.py b/catalyst/targets/netboot2.py
index e9e2625..7edfc1f 100644
--- a/catalyst/targets/netboot2.py
+++ b/catalyst/targets/netboot2.py
@@ -10,6 +10,7 @@ from stat import ST_UID, ST_GID, ST_MODE
from catalyst.support import (CatalystError, normpath,
touch, cmd, list_bashify)
+from catalyst.fileops import ensure_dirs
from catalyst.base.stagebase import StageBase
@@ -62,8 +63,7 @@ class netboot2(StageBase):
"Could not remove existing file: "+self.settings["target_path"],env=self.env)
touch(self.settings["autoresume_path"]+"setup_target_path")
- if not os.path.exists(self.settings["storedir"]+"/builds/"):
- os.makedirs(self.settings["storedir"]+"/builds/")
+ ensure_dirs(self.settings["storedir"]+"/builds/")
def copy_files_to_image(self):
# copies specific files from the buildroot to merge_path
@@ -158,7 +158,7 @@ class netboot2(StageBase):
# the proper perms and ownership
mystat=os.stat(myemp)
shutil.rmtree(myemp)
- os.makedirs(myemp,0755)
+ ensure_dirs(myemp, mode=0755)
os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
os.chmod(myemp,mystat[ST_MODE])
touch(self.settings["autoresume_path"]+"empty")
diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index 6c2396e..1887110 100644
--- a/catalyst/targets/snapshot.py
+++ b/catalyst/targets/snapshot.py
@@ -10,7 +10,7 @@ from stat import ST_UID, ST_GID, ST_MODE
from catalyst.support import normpath, cmd
from catalyst.base.targetbase import TargetBase
from catalyst.base.genbase import GenBase
-
+from catalyst.fileops import ensure_dirs
class snapshot(TargetBase, GenBase):
"""
Builder class for snapshots.
@@ -31,8 +31,7 @@ class snapshot(TargetBase, GenBase):
def setup(self):
x=normpath(self.settings["storedir"]+"/snapshots")
- if not os.path.exists(x):
- os.makedirs(x)
+ ensure_dirs(x)
def mount_safety_check(self):
pass
@@ -50,8 +49,7 @@ class snapshot(TargetBase, GenBase):
" from "+self.settings["portdir"]+"..."
mytmp=self.settings["tmp_path"]
- if not os.path.exists(mytmp):
- os.makedirs(mytmp)
+ ensure_dirs(mytmp)
cmd("rsync -a --delete --exclude /packages/ --exclude /distfiles/ " +
"--exclude /local/ --exclude CVS/ --exclude .svn --filter=H_**/files/digest-* " +
@@ -88,6 +86,6 @@ class snapshot(TargetBase, GenBase):
if os.uname()[0] == "FreeBSD":
os.system("chflags -R noschg "+myemp)
shutil.rmtree(myemp)
- os.makedirs(myemp,0755)
+ ensure_dirs(myemp, mode=0755)
os.chown(myemp,mystat[ST_UID],mystat[ST_GID])
os.chmod(myemp,mystat[ST_MODE])
diff --git a/catalyst/targets/stage1.py b/catalyst/targets/stage1.py
index e72cbcc..3d3b9e9 100644
--- a/catalyst/targets/stage1.py
+++ b/catalyst/targets/stage1.py
@@ -6,6 +6,7 @@ stage1 target
import os
from catalyst.support import normpath, list_to_string
+from catalyst.fileops import ensure_dirs
from catalyst.base.stagebase import StageBase
@@ -64,10 +65,8 @@ class stage1(StageBase):
# baselayout no longer creates the .keep files in proc and dev for FreeBSD as it
# would create them too late...we need them earlier before bind mounting filesystems
# since proc and dev are not writeable, so...create them here
- if not os.path.exists(self.settings["stage_path"]+"/proc"):
- os.makedirs(self.settings["stage_path"]+"/proc")
- if not os.path.exists(self.settings["stage_path"]+"/dev"):
- os.makedirs(self.settings["stage_path"]+"/dev")
+ ensure_dirs(self.settings["stage_path"]+"/proc")
+ ensure_dirs(self.settings["stage_path"]+"/dev")
if not os.path.isfile(self.settings["stage_path"]+"/proc/.keep"):
try:
proc_keepfile = open(self.settings["stage_path"]+"/proc/.keep","w")
@@ -87,8 +86,7 @@ class stage1(StageBase):
def set_mounts(self):
# stage_path/proc probably doesn't exist yet, so create it
- if not os.path.exists(self.settings["stage_path"]+"/proc"):
- os.makedirs(self.settings["stage_path"]+"/proc")
+ ensure_dirs(self.settings["stage_path"]+"/proc")
# alter the mount mappings to bind mount proc onto it
self.mounts.append("stage1root/proc")
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/, catalyst/base/
@ 2015-09-06 21:18 Brian Dolbec
2015-09-06 21:21 ` [gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/, catalyst/targets/ Brian Dolbec
0 siblings, 1 reply; 4+ messages in thread
From: Brian Dolbec @ 2015-09-06 21:18 UTC (permalink / raw
To: gentoo-commits
commit: 99e9ceabe0532f5fb1b12ab55d35097b5be3cfa8
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jun 15 16:33:29 2015 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Sep 6 21:18:00 2015 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=99e9ceab
DeComp bug fixes
catalyst/base/genbase.py | 2 +-
catalyst/base/stagebase.py | 14 +++++++++-----
catalyst/main.py | 5 +++--
catalyst/targets/snapshot.py | 34 +++++++++++++++++++++++++---------
4 files changed, 38 insertions(+), 17 deletions(-)
diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index e818781..c05b36d 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -24,7 +24,7 @@ class GenBase(object):
array=keys.keys()
array.sort()
for j in array:
- contents = contents_map.generate_contents(file, j,
+ contents = contents_map.contents(file, j,
verbose="VERBOSE" in self.settings)
if contents:
myf.write(contents)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index ea37591..7212409 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -10,7 +10,7 @@ from stat import ST_UID, ST_GID, ST_MODE
# for convienience
pjoin = os.path.join
-from decomp.compress import CompressMap
+from DeComp.compress import CompressMap
from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
PORT_LOGDIR_CLEAN)
@@ -38,10 +38,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.required_values.extend(["version_stamp","target","subarch",\
"rel_type","profile","snapshot","source_subpath"])
- self.valid_values.extend(["version_stamp","target","subarch",\
- "rel_type","profile","snapshot","source_subpath","portage_confdir",\
- "cflags","cxxflags","fcflags","fflags","ldflags","cbuild","hostuse","portage_overlay",\
- "distcc_hosts","makeopts","pkgcache_path","kerncache_path"])
+ self.valid_values.extend(["version_stamp","target","subarch",
+ "rel_type","profile","snapshot","source_subpath","portage_confdir",
+ "cflags","cxxflags","fcflags","fflags","ldflags","cbuild","hostuse","portage_overlay",
+ "distcc_hosts","makeopts","pkgcache_path","kerncache_path",
+ "compression_mode", "decompression_mode"])
self.set_valid_build_kernel_vars(addlargs)
TargetBase.__init__(self, myspec, addlargs)
@@ -1371,6 +1372,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
# lists in arch/amd64.py and friends. If we wanted self.settigs[var]
# of any depth, we should make this function recursive.
elif type(self.settings[x]) == types.DictType:
+ if x in ["compress_definitions",
+ "decompress_definitions"]:
+ continue
self.env[varname] = string.join(self.settings[x].keys())
for y in self.settings[x].keys():
varname2 = "clst_"+string.replace(y,"/","_")
diff --git a/catalyst/main.py b/catalyst/main.py
index 7e3b55e..dc9342b 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -13,11 +13,12 @@ import os.path
__selfpath__ = os.path.abspath(os.path.dirname(__file__))
-from decomp.compress import COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS
+from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,
+ CONTENTS_DEFINITIONS)
+from DeComp.contents import ContentsMap
import catalyst.config
import catalyst.util
-from catalyst.contents import ContentsMap, CONTENTS_DEFINITIONS
from catalyst.defaults import confdefaults, option_messages
from catalyst.hash_utils import HashMap, HASH_DEFINITIONS
from catalyst.lock import LockInUse
diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index 1887110..ad96d37 100644
--- a/catalyst/targets/snapshot.py
+++ b/catalyst/targets/snapshot.py
@@ -6,18 +6,21 @@ import os
import shutil
from stat import ST_UID, ST_GID, ST_MODE
+from DeComp.compress import CompressMap
from catalyst.support import normpath, cmd
from catalyst.base.targetbase import TargetBase
from catalyst.base.genbase import GenBase
from catalyst.fileops import ensure_dirs
+
+
class snapshot(TargetBase, GenBase):
"""
Builder class for snapshots.
"""
def __init__(self,myspec,addlargs):
self.required_values=["version_stamp","target"]
- self.valid_values=["version_stamp","target"]
+ self.valid_values=["version_stamp","target", "compression_mode"]
TargetBase.__init__(self, myspec, addlargs)
GenBase.__init__(self,myspec)
@@ -26,7 +29,7 @@ class snapshot(TargetBase, GenBase):
st=self.settings["storedir"]
self.settings["snapshot_path"] = normpath(st + "/snapshots/"
+ self.settings["snapshot_name"]
- + self.settings["version_stamp"] + ".tar.bz2")
+ + self.settings["version_stamp"])
self.settings["tmp_path"]=normpath(st+"/tmp/"+self.settings["target_subpath"])
def setup(self):
@@ -51,18 +54,31 @@ class snapshot(TargetBase, GenBase):
mytmp=self.settings["tmp_path"]
ensure_dirs(mytmp)
+ target_snapshot = self.settings["portdir"] + "/ " + mytmp + "/%s/" % self.settings["repo_name"]
cmd("rsync -a --delete --exclude /packages/ --exclude /distfiles/ " +
"--exclude /local/ --exclude CVS/ --exclude .svn --filter=H_**/files/digest-* " +
- self.settings["portdir"] + "/ " + mytmp + "/%s/" % self.settings["repo_name"],
+ target_snapshot,
"Snapshot failure", env=self.env)
print "Compressing Portage snapshot tarball..."
- cmd("tar -I lbzip2 -cf " + self.settings["snapshot_path"] + " -C " +
- mytmp + " " + self.settings["repo_name"],
- "Snapshot creation failure",env=self.env)
-
- self.gen_contents_file(self.settings["snapshot_path"])
- self.gen_digest_file(self.settings["snapshot_path"])
+ compressor = CompressMap(self.settings["compress_definitions"],
+ env=self.env, default_mode=self.settings['compression_mode'])
+ infodict = compressor.create_infodict(
+ source=self.settings["repo_name"],
+ destination=self.settings["snapshot_path"],
+ basedir=mytmp,
+ filename=self.settings["snapshot_path"],
+ mode=self.settings["compression_mode"],
+ auto_extension=True
+ )
+ if not compressor.compress(infodict):
+ print("Snapshot compression failure")
+ else:
+ filename = '.'.join([self.settings["snapshot_path"],
+ compressor.extension(self.settings["compression_mode"])])
+ print("COMPRESSOR success!!!! filename", filename)
+ self.gen_contents_file(filename)
+ self.gen_digest_file(filename)
self.cleanup()
print "snapshot: complete!"
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/, catalyst/targets/
2015-09-06 21:18 [gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/, catalyst/base/ Brian Dolbec
@ 2015-09-06 21:21 ` Brian Dolbec
0 siblings, 0 replies; 4+ messages in thread
From: Brian Dolbec @ 2015-09-06 21:21 UTC (permalink / raw
To: gentoo-commits
commit: 99e9ceabe0532f5fb1b12ab55d35097b5be3cfa8
Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Mon Jun 15 16:33:29 2015 +0000
Commit: Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Sun Sep 6 21:18:00 2015 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=99e9ceab
DeComp bug fixes
catalyst/base/genbase.py | 2 +-
catalyst/base/stagebase.py | 14 +++++++++-----
catalyst/main.py | 5 +++--
catalyst/targets/snapshot.py | 34 +++++++++++++++++++++++++---------
4 files changed, 38 insertions(+), 17 deletions(-)
diff --git a/catalyst/base/genbase.py b/catalyst/base/genbase.py
index e818781..c05b36d 100644
--- a/catalyst/base/genbase.py
+++ b/catalyst/base/genbase.py
@@ -24,7 +24,7 @@ class GenBase(object):
array=keys.keys()
array.sort()
for j in array:
- contents = contents_map.generate_contents(file, j,
+ contents = contents_map.contents(file, j,
verbose="VERBOSE" in self.settings)
if contents:
myf.write(contents)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index ea37591..7212409 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -10,7 +10,7 @@ from stat import ST_UID, ST_GID, ST_MODE
# for convienience
pjoin = os.path.join
-from decomp.compress import CompressMap
+from DeComp.compress import CompressMap
from catalyst.defaults import (SOURCE_MOUNT_DEFAULTS, TARGET_MOUNT_DEFAULTS,
PORT_LOGDIR_CLEAN)
@@ -38,10 +38,11 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.required_values.extend(["version_stamp","target","subarch",\
"rel_type","profile","snapshot","source_subpath"])
- self.valid_values.extend(["version_stamp","target","subarch",\
- "rel_type","profile","snapshot","source_subpath","portage_confdir",\
- "cflags","cxxflags","fcflags","fflags","ldflags","cbuild","hostuse","portage_overlay",\
- "distcc_hosts","makeopts","pkgcache_path","kerncache_path"])
+ self.valid_values.extend(["version_stamp","target","subarch",
+ "rel_type","profile","snapshot","source_subpath","portage_confdir",
+ "cflags","cxxflags","fcflags","fflags","ldflags","cbuild","hostuse","portage_overlay",
+ "distcc_hosts","makeopts","pkgcache_path","kerncache_path",
+ "compression_mode", "decompression_mode"])
self.set_valid_build_kernel_vars(addlargs)
TargetBase.__init__(self, myspec, addlargs)
@@ -1371,6 +1372,9 @@ class StageBase(TargetBase, ClearBase, GenBase):
# lists in arch/amd64.py and friends. If we wanted self.settigs[var]
# of any depth, we should make this function recursive.
elif type(self.settings[x]) == types.DictType:
+ if x in ["compress_definitions",
+ "decompress_definitions"]:
+ continue
self.env[varname] = string.join(self.settings[x].keys())
for y in self.settings[x].keys():
varname2 = "clst_"+string.replace(y,"/","_")
diff --git a/catalyst/main.py b/catalyst/main.py
index 7e3b55e..dc9342b 100644
--- a/catalyst/main.py
+++ b/catalyst/main.py
@@ -13,11 +13,12 @@ import os.path
__selfpath__ = os.path.abspath(os.path.dirname(__file__))
-from decomp.compress import COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS
+from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS,
+ CONTENTS_DEFINITIONS)
+from DeComp.contents import ContentsMap
import catalyst.config
import catalyst.util
-from catalyst.contents import ContentsMap, CONTENTS_DEFINITIONS
from catalyst.defaults import confdefaults, option_messages
from catalyst.hash_utils import HashMap, HASH_DEFINITIONS
from catalyst.lock import LockInUse
diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index 1887110..ad96d37 100644
--- a/catalyst/targets/snapshot.py
+++ b/catalyst/targets/snapshot.py
@@ -6,18 +6,21 @@ import os
import shutil
from stat import ST_UID, ST_GID, ST_MODE
+from DeComp.compress import CompressMap
from catalyst.support import normpath, cmd
from catalyst.base.targetbase import TargetBase
from catalyst.base.genbase import GenBase
from catalyst.fileops import ensure_dirs
+
+
class snapshot(TargetBase, GenBase):
"""
Builder class for snapshots.
"""
def __init__(self,myspec,addlargs):
self.required_values=["version_stamp","target"]
- self.valid_values=["version_stamp","target"]
+ self.valid_values=["version_stamp","target", "compression_mode"]
TargetBase.__init__(self, myspec, addlargs)
GenBase.__init__(self,myspec)
@@ -26,7 +29,7 @@ class snapshot(TargetBase, GenBase):
st=self.settings["storedir"]
self.settings["snapshot_path"] = normpath(st + "/snapshots/"
+ self.settings["snapshot_name"]
- + self.settings["version_stamp"] + ".tar.bz2")
+ + self.settings["version_stamp"])
self.settings["tmp_path"]=normpath(st+"/tmp/"+self.settings["target_subpath"])
def setup(self):
@@ -51,18 +54,31 @@ class snapshot(TargetBase, GenBase):
mytmp=self.settings["tmp_path"]
ensure_dirs(mytmp)
+ target_snapshot = self.settings["portdir"] + "/ " + mytmp + "/%s/" % self.settings["repo_name"]
cmd("rsync -a --delete --exclude /packages/ --exclude /distfiles/ " +
"--exclude /local/ --exclude CVS/ --exclude .svn --filter=H_**/files/digest-* " +
- self.settings["portdir"] + "/ " + mytmp + "/%s/" % self.settings["repo_name"],
+ target_snapshot,
"Snapshot failure", env=self.env)
print "Compressing Portage snapshot tarball..."
- cmd("tar -I lbzip2 -cf " + self.settings["snapshot_path"] + " -C " +
- mytmp + " " + self.settings["repo_name"],
- "Snapshot creation failure",env=self.env)
-
- self.gen_contents_file(self.settings["snapshot_path"])
- self.gen_digest_file(self.settings["snapshot_path"])
+ compressor = CompressMap(self.settings["compress_definitions"],
+ env=self.env, default_mode=self.settings['compression_mode'])
+ infodict = compressor.create_infodict(
+ source=self.settings["repo_name"],
+ destination=self.settings["snapshot_path"],
+ basedir=mytmp,
+ filename=self.settings["snapshot_path"],
+ mode=self.settings["compression_mode"],
+ auto_extension=True
+ )
+ if not compressor.compress(infodict):
+ print("Snapshot compression failure")
+ else:
+ filename = '.'.join([self.settings["snapshot_path"],
+ compressor.extension(self.settings["compression_mode"])])
+ print("COMPRESSOR success!!!! filename", filename)
+ self.gen_contents_file(filename)
+ self.gen_digest_file(filename)
self.cleanup()
print "snapshot: complete!"
^ permalink raw reply related [flat|nested] 4+ messages in thread
* [gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/, catalyst/targets/
@ 2015-10-06 15:31 Mike Frysinger
0 siblings, 0 replies; 4+ messages in thread
From: Mike Frysinger @ 2015-10-06 15:31 UTC (permalink / raw
To: gentoo-commits
commit: d5fc0982052ada0eaab209b8c56574220f63a698
Author: Mike Frysinger <vapier <AT> gentoo <DOT> org>
AuthorDate: Tue Oct 6 13:57:12 2015 +0000
Commit: Mike Frysinger <vapier <AT> gentoo <DOT> org>
CommitDate: Tue Oct 6 13:57:12 2015 +0000
URL: https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=d5fc0982
lint: convert funcs to @staticmethod where it makes sense
These functions don't actually utilize |self|, so make them into
@staticmethod's to quiet down the linter.
catalyst/base/stagebase.py | 3 ++-
catalyst/lock.py | 9 ++++++---
catalyst/targets/snapshot.py | 3 ++-
3 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index b9dd1d5..409fcab 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -1695,7 +1695,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
self.unbind()
raise CatalystError("build aborting due to livecd_update error.")
- def _debug_pause_(self):
+ @staticmethod
+ def _debug_pause_():
py_input("press any key to continue: ")
# vim: ts=4 sw=4 sta et sts=4 ai
diff --git a/catalyst/lock.py b/catalyst/lock.py
index 71918d6..01b1aa8 100644
--- a/catalyst/lock.py
+++ b/catalyst/lock.py
@@ -320,7 +320,8 @@ class LockDir(object):
del self.hardlock_paths[self.lockdir]
print self.hardlock_paths
- def hardlock_name(self, path):
+ @staticmethod
+ def hardlock_name(path):
mypath=path+"/.hardlock-"+os.uname()[1]+"-"+str(os.getpid())
newpath = os.path.normpath(mypath)
if len(newpath) > 1:
@@ -328,7 +329,8 @@ class LockDir(object):
newpath = "/"+newpath.lstrip("/")
return newpath
- def hardlink_is_mine(self,link,lock):
+ @staticmethod
+ def hardlink_is_mine(link, lock):
import stat
try:
myhls = os.stat(link)
@@ -347,7 +349,8 @@ class LockDir(object):
return True
return False
- def hardlink_active(self, lock):
+ @staticmethod
+ def hardlink_active(lock):
if not os.path.exists(lock):
return False
diff --git a/catalyst/targets/snapshot.py b/catalyst/targets/snapshot.py
index 87340b7..a117a21 100644
--- a/catalyst/targets/snapshot.py
+++ b/catalyst/targets/snapshot.py
@@ -90,7 +90,8 @@ class snapshot(TargetBase, GenBase):
def kill_chroot_pids(self):
pass
- def cleanup(self):
+ @staticmethod
+ def cleanup():
print "Cleaning up..."
def purge(self):
^ permalink raw reply related [flat|nested] 4+ messages in thread
end of thread, other threads:[~2015-10-06 15:31 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2015-09-06 21:18 [gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/targets/, catalyst/base/ Brian Dolbec
2015-09-06 21:21 ` [gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/, catalyst/targets/ Brian Dolbec
-- strict thread matches above, loose matches on Subject: below --
2015-10-06 15:31 Mike Frysinger
2015-01-01 5:59 [gentoo-commits] proj/catalyst:pending commit in: catalyst/, catalyst/base/, catalyst/targets/ Brian Dolbec
2015-02-26 20:12 ` [gentoo-commits] proj/catalyst:master commit in: catalyst/base/, catalyst/, catalyst/targets/ Brian Dolbec
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox