diff --git a/src/machine_admin/boot_image/deb_preseed.py b/src/machine_admin/boot_image/deb_preseed.py index b1161d9..c318b7d 100644 --- a/src/machine_admin/boot_image/deb_preseed.py +++ b/src/machine_admin/boot_image/deb_preseed.py @@ -6,6 +6,7 @@ from machine_admin.util import run_op logger = logging.getLogger(__name__) + class DebPreseeder: def __init__(self, iso_path: Path, preseed_path: Path): @@ -15,7 +16,7 @@ class DebPreseeder: self.udevil = SystemPackage("udevil") self.gzip = SystemPackage("gzip") self.mount_point = None - + def mount_iso(self): self.udevil.check_available() op = f"udevil mount {self.iso_path}" @@ -50,7 +51,7 @@ class DebPreseeder: def regenerate_checksum(self): checksum_path = f"{self.workdir}/isofiles/md5sum.txt" - run_op(f"chmod +w {checksum_path}") + run_op(f"chmod +w {checksum_path}") def regenerate_iso(self, workdir: Path): @@ -59,16 +60,3 @@ class DebPreseeder: self.add_preseed_file() self.regenerate_checksum() - - - - - - - - - - - - - diff --git a/src/machine_admin/firewall.py b/src/machine_admin/firewall.py index 3180eb6..6859b72 100644 --- a/src/machine_admin/firewall.py +++ b/src/machine_admin/firewall.py @@ -4,6 +4,7 @@ from .util import run_op logger = logging.getLogger(__name__) + class UfwInterface: def __init__(self): @@ -12,13 +13,14 @@ class UfwInterface: def enable(self): op = "ufw enable" logger.info(f"Enabling ufw: {op}") - run_op(op) + run_op(op) def allow_app(self, app_name: str): op = f"ufw allow {app_name}" logger.info(f"Allowing ufw app: {op}") run_op(op) + class Firewall: def __init__(self): @@ -28,4 +30,4 @@ class Firewall: self.ufw.allow_app(app_name) def enable(self): - self.ufw.enable() \ No newline at end of file + self.ufw.enable() diff --git a/src/machine_admin/machine.py b/src/machine_admin/machine.py index 3001e40..a67c018 100644 --- a/src/machine_admin/machine.py +++ b/src/machine_admin/machine.py @@ -3,6 +3,7 @@ from .user import User, UserManager from .ssh_config import SshConfig from .package_manager import PackageManager + class Machine: def __init__(self, default_user: User): @@ -26,4 +27,4 @@ class Machine: self.enable_firewall() self.secure_ssh_config() self.package_manager.install_packages(["rsync", "fail2ban"]) - self.ssh_config.copy_ssh_dir_to_user(self.user.name) \ No newline at end of file + self.ssh_config.copy_ssh_dir_to_user(self.user.name) diff --git a/src/machine_admin/package_manager.py b/src/machine_admin/package_manager.py index 288518a..29b2c53 100644 --- a/src/machine_admin/package_manager.py +++ b/src/machine_admin/package_manager.py @@ -5,10 +5,11 @@ from .util import run_op, has_program logger = logging.getLogger(__name__) + class AptInterface: def __init__(self): - pass + pass def update(self): op = "apt-get update" @@ -26,6 +27,7 @@ class AptInterface: logger.info(f"Installing packages: {op}") run_op(op) + class SystemPackage: def __init__(self, name): @@ -40,7 +42,7 @@ class SystemPackage: if Distro.has_apt(): msg = f"Program not found. Install with: sudo apt-get install {self.name}" raise RuntimeError(msg) - + class PackageManager: diff --git a/src/machine_admin/platform/distro.py b/src/machine_admin/platform/distro.py index 63482bf..f2474f3 100644 --- a/src/machine_admin/platform/distro.py +++ b/src/machine_admin/platform/distro.py @@ -1,11 +1,12 @@ from machine_admin.util import has_program + class Distro: def __init__(self, name): - self.name = name + self.name = name self.version = "" @staticmethod def has_apt(self): - return has_program("apt-get") \ No newline at end of file + return has_program("apt-get") diff --git a/src/machine_admin/platform/linux.py b/src/machine_admin/platform/linux.py index 0b3875c..3da8b03 100644 --- a/src/machine_admin/platform/linux.py +++ b/src/machine_admin/platform/linux.py @@ -2,5 +2,6 @@ import sys from .distro import Distro + def is_linux(): - return sys.platform == "linux" or sys.platform == "linux2" \ No newline at end of file + return sys.platform == "linux" or sys.platform == "linux2" diff --git a/src/machine_admin/ssh_config.py b/src/machine_admin/ssh_config.py index ad0b8d9..ef9f9c9 100644 --- a/src/machine_admin/ssh_config.py +++ b/src/machine_admin/ssh_config.py @@ -5,14 +5,17 @@ from .util import run_op logger = logging.getLogger(__name__) + class SshConfig: def __init__(self): self.config_path = Path("/etc/ssh/ssh_config") - self.target_values = {"PermitRootLogin": "No", - "PasswordAuthentication": "No", - "ChallengeResponseAuthentication": "No", - "UsePAM": "No"} + self.target_values = { + "PermitRootLogin": "No", + "PasswordAuthentication": "No", + "ChallengeResponseAuthentication": "No", + "UsePAM": "No", + } def sync_target_values(self): logger.info(f"Updating ssh config in: {self.config_path}") @@ -23,8 +26,7 @@ class SshConfig: logger.info(f"Restarting ssh service: {op}") run_op(op) - def copy_ssh_dir_to_user(self, username:str): + def copy_ssh_dir_to_user(self, username: str): op = f"rsync --archive --chown={username}:{username} ~/.ssh /home/{username}" logger.info(f"Copying ssh dir to user: {op}") run_op(op) - \ No newline at end of file diff --git a/src/machine_admin/user.py b/src/machine_admin/user.py index f51bd90..998caa0 100644 --- a/src/machine_admin/user.py +++ b/src/machine_admin/user.py @@ -1,13 +1,15 @@ -import logging +import logging from .util import run_op logger = logging.getLogger(__name__) + class User: def __init__(self, name, has_sudo=False): - self.name = name + self.name = name self.has_sudo = has_sudo + class UserManager: def __init__(self): @@ -17,7 +19,7 @@ class UserManager: self.add_user(user) if user.has_sudo: self.add_user_to_sudo(user) - + def add_user(self, user: User): op = f'adduser {user.name} --disabled-password --comment ""' logger.info(f"Adding user: {op}") @@ -26,4 +28,4 @@ class UserManager: def add_user_to_sudo(self, user: User): op = f"usermod -aG sudo {user.name}" logger.info(f"Adding user to sudo: {op}") - run_op(op) \ No newline at end of file + run_op(op) diff --git a/src/machine_admin/util.py b/src/machine_admin/util.py index 5d05154..a50eb87 100644 --- a/src/machine_admin/util.py +++ b/src/machine_admin/util.py @@ -5,30 +5,27 @@ logger = logging.getLogger(__name__) _DRY_RUN = False + def set_is_dry_run(is_dry_run: bool): _DRY_RUN = is_dry_run -def run_op(op: str, capture_output:bool = False, - cwd:str = None): + +def run_op(op: str, capture_output: bool = False, cwd: str = None): if not _DRY_RUN: if capture_output: - ret = subprocess.run(op, - shell=True, - capture_output = True, - text = True, - cwd=cwd) + ret = subprocess.run( + op, shell=True, capture_output=True, text=True, cwd=cwd + ) ret.check_returncode() return ret.stdout else: - ret = subprocess.run(op, shell=True, cwd=cwd) + ret = subprocess.run(op, shell=True, cwd=cwd) ret.check_returncode() else: logger.info(f"Dry Run | {op}") + def has_program(program_name: str): op = f"which {program_name}" - ret = subprocess.run(op, - shell=True, - capture_output = True, - text = True) - return len(ret.stdout) \ No newline at end of file + ret = subprocess.run(op, shell=True, capture_output=True, text=True) + return len(ret.stdout) diff --git a/src/machine_setup.py b/src/machine_setup.py index f2ac28d..79ea841 100644 --- a/src/machine_setup.py +++ b/src/machine_setup.py @@ -10,22 +10,22 @@ logger = logging.getLogger(__name__) if __name__ == "__main__": parser = argparse.ArgumentParser( - prog='MachineSetup', - description='Scripts for machine provisioning') + prog="MachineSetup", description="Scripts for machine provisioning" + ) - parser.add_argument('username', - help="Name of the default non-root user") - parser.add_argument('--dry_run', + parser.add_argument("username", help="Name of the default non-root user") + parser.add_argument( + "--dry_run", help="If set then don't change the system state - used for testing.", - default = False) + default=False, + ) args = parser.parse_args() set_is_dry_run(args.dry_run) logging.basicConfig() logging.getLogger().setLevel(logging.INFO) - + user = User(args.username, has_sudo=True) machine = Machine(user) machine.setup() - diff --git a/src/music/convert.py b/src/music/convert.py index 8befcf2..762f2a6 100644 --- a/src/music/convert.py +++ b/src/music/convert.py @@ -3,59 +3,74 @@ import os import logging import shutil from pathlib import Path +from typing import NamedTuple import subprocess import uuid from multiprocessing import Pool logger = logging.getLogger(__name__) + def convert_audio_file(args): cmd, output_tmp, output_path = args - + subprocess.run(cmd, shell=True) os.makedirs(output_path.parent, exist_ok=True) shutil.move(output_tmp, output_path) - -class AudioFileConverter(): - def __init__(self, input_dir: Path, output_dir: Path): - self.input_dir = input_dir - self.output_dir = output_dir - self.output_fmt = "mp3" - self.input_fmt = "flac" - def get_converted_path(self, input_path: Path): - relative_path = input_path.relative_to(self.input_dir) - output_filename = str(input_path.stem) + f".{self.output_fmt}" - output_filename.replace("'", "") - output_path = self.output_dir / relative_path.parent / output_filename - return output_path - - def convert(self): +class ConversionConfig(NamedTuple): + input_dir: Path + output_dir: Path + output_fmt: str + input_fmt: str - logger.info("Converting files in %s", self.input_dir) - logger.info("Writing output to: %s", self.output_dir) - os.makedirs(self.output_dir, exist_ok=True) +def _get_converted_path(input_path: Path, config: ConversionConfig): + """ + Return the path you would obtain if you moved the file in the input + path to the output directory in the config and changed its extension. + """ - input_files = list(self.input_dir.resolve().rglob(f"*.{self.input_fmt}")) - output_files = [] - for input_file in input_files: - candidate_output = self.get_converted_path(input_file) - if not candidate_output.exists(): - output_files.append(candidate_output) - - tasks = [] - for idx, (input_path, output_path) in enumerate(zip(input_files, output_files)): + relative_path = input_path.relative_to(config.input_dir) + output_filename = str(input_path.stem) + f".{config.output_fmt}" + output_filename.replace("'", "") + output_path = config.output_dir / relative_path.parent / output_filename + return output_path - logger.info("Converting file %d of %d", idx, len(input_files)) - output_tmp = self.output_dir / (str(uuid.uuid4()) + f".{self.output_fmt}") - cmd = f"ffmpeg -i '{input_path}' -ab 320k -map_metadata 0 -id3v2_version 3 '{output_tmp}'" - tasks.append((cmd, output_tmp, output_path)) +def get_files_recursive(search_path: Path, extension: str): + return list(search_path.rglob(f"*.{extension}")) - with Pool(10) as p: - p.map(convert_audio_file, tasks) + +def convert(config: ConversionConfig): + + logger.info("Converting files in %s", config.input_dir) + logger.info("Writing output to: %s", config.output_dir) + + os.makedirs(config.output_dir, exist_ok=True) + + input_files = get_files_recursive( + config.input_dir.resolve(), config.input_fmt + ) + output_files = [] + for input_file in input_files: + candidate_output = _get_converted_path(input_file) + if not candidate_output.exists(): + output_files.append(candidate_output) + print(output_files) + + tasks = [] + for idx, (input_path, output_path) in enumerate(zip(input_files, output_files)): + + logger.info("Converting file %d of %d", idx, len(input_files)) + + output_tmp = self.output_dir / (str(uuid.uuid4()) + f".{self.output_fmt}") + cmd = f"ffmpeg -i '{input_path}' -ab 320k -map_metadata 0 -id3v2_version 3 '{output_tmp}'" + tasks.append((cmd, output_tmp, output_path)) + + with Pool(10) as p: + p.map(convert_audio_file, tasks) def move(input_dir: Path, output_dir: Path): @@ -65,25 +80,27 @@ def move(input_dir: Path, output_dir: Path): mp3_files = list(input_dir.resolve().rglob("*.mp3")) for idx, path in enumerate(mp3_files): - logger.info("Moving file %d of %d", idx, len(mp3_files)) + logger.info("Moving file %d of %d", idx, len(mp3_files)) relative_path = path.relative_to(input_dir) output_path = output_dir / relative_path os.makedirs(output_path.parent, exist_ok=True) shutil.move(path, output_path) - + + if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--input_dir', - type=Path, - default=Path(), - help="Directory with input files for conversion.") + parser.add_argument( + "--input_dir", + type=Path, + default=Path(), + help="Directory with input files for conversion.", + ) - parser.add_argument('--output_dir', - type=Path, - default=Path(), - help="Directory for converted files") + parser.add_argument( + "--output_dir", type=Path, default=Path(), help="Directory for converted files" + ) args = parser.parse_args() @@ -92,6 +109,4 @@ if __name__ == "__main__": converter = AudioFileConverter(args.input_dir.resolve(), args.output_dir.resolve()) converter.convert() - #move(args.input_dir.resolve(), args.output_dir.resolve()) - - + # move(args.input_dir.resolve(), args.output_dir.resolve()) diff --git a/src/network/sync.py b/src/network/sync.py index 7b8da7c..37fd697 100644 --- a/src/network/sync.py +++ b/src/network/sync.py @@ -28,37 +28,43 @@ def sync(source_dir: Path, target_dir: Path, host: str): source_files = [] for dir_path, _, files in os.walk(source_dir): for each_file in files: - source_files.append(Path(dir_path + "/" + each_file).relative_to(source_dir)) - + source_files.append( + Path(dir_path + "/" + each_file).relative_to(source_dir) + ) + sync_files = [] for source_file in source_files: if source_file not in target_files: sync_files.append(source_file) - sync_dir = Path(os.getcwd())/"sync" + sync_dir = Path(os.getcwd()) / "sync" os.makedirs(sync_dir, exist_ok=True) for sync_file in sync_files: - os.makedirs((sync_dir/sync_file).parent, exist_ok=True) + os.makedirs((sync_dir / sync_file).parent, exist_ok=True) shutil.copy(source_dir / sync_file, sync_dir / sync_file) - + + if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--source_dir', - type=Path, - default=Path(), - help="Directory with source files to sync from.") + parser.add_argument( + "--source_dir", + type=Path, + default=Path(), + help="Directory with source files to sync from.", + ) - parser.add_argument('--host', - type=str, - default = "", - help="Name of host system to sync with.") - - parser.add_argument('--target_dir', - type=Path, - default=Path(), - help="Directory with source files to sync to.") + parser.add_argument( + "--host", type=str, default="", help="Name of host system to sync with." + ) + + parser.add_argument( + "--target_dir", + type=Path, + default=Path(), + help="Directory with source files to sync to.", + ) args = parser.parse_args() @@ -66,5 +72,3 @@ if __name__ == "__main__": logging.getLogger().setLevel(logging.INFO) sync(args.source_dir.resolve(), args.target_dir.resolve(), args.host) - -