diff --git a/Containerfile b/Containerfile index 8e96c64..5b86fa3 100644 --- a/Containerfile +++ b/Containerfile @@ -1,4 +1,6 @@ FROM debian +RUN apt-get update; apt-get install -y python3 + COPY src/machine_admin /machine_admin COPY src/machine_setup.py / diff --git a/README.md b/README.md index 05ebea2..bc510ec 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,13 @@ -This repo is a collection of ad-hoc scripts and snippets for setting up and administering a small cloud server. They are just a stepping stone toward for formal automation. +This repo has some scripts and small notes for setting up development and deployment environments on different platforms. -# Machine setup +* [Mac Development](./doc/mac.md) +* [Emacs](./doc/emacs_notes.md) +* [VM Notes](./doc/vm_notes.md) -```sh -./setup_machine.sh -``` +There is another repo just for dotfiles that may be of interest also: https://codeberg.org/jmsgrogan/dotfiles -# Set up basic nginx server +# Copyright -```sh -./setup_basic_nginx.sh -``` +The contents of this repo are Copyright James Grogan 2024. Software is licensed under the AGPL. See the included LICENSE file for details. +Documentation and images can be re-used under a Creative Commons CC BY-SA license: https://creativecommons.org/licenses/by-sa/4.0/ diff --git a/doc/emacs_notes.md b/doc/emacs_notes.md new file mode 100644 index 0000000..15a2e8c --- /dev/null +++ b/doc/emacs_notes.md @@ -0,0 +1,69 @@ + +# Emacs + +## Base + +* `C-x C-s`: Save current file. +* `C-x C-f`: Find or open file. + +## Markdown + +[Module Reference](http://www.mplayerhq.hu/design7/news.html) + +* `C-c C-s 1`: Insert level one heading with hash delimiter. +* `C-c C-c p`: Show preview in browser. +* `C-c C-x l`: Toggle url hiding +* `C-c C-x i`: Toggle inline images +* `C-c C-x m`: Toggle markdown visibility +* `S-Tab`: Toggle global visibility +* `TAB`: Toggle local header visibility + + +## Evil + +[Reference card](https://michaelgoerz.net/refcards/vimqrc.pdf) + +* `CTRL- {D, U, F, B}`: Scroll down/up by half or full page +* `zt zz zb`: Scroll current line to top, centre, bottom +* `0 $`: Move to beginning or end of line +* `{ }`: Move to beginning or end of paragraph +* `h l j k`: Character left, right, up, down +* `y p`: Yank (copy) or paste + +## Magit + +[Reference card](https://magit.vc/manual/magit-refcard.pdf) + +* `CTRL-x g`: Status buffer +* `S`: Stage all changes +* `c`: Start a commit + * `C-c C-c`: Finish a commit +* `P`: Start a push + * `u`: Push to origin + +## Auctex + +## Org Mode ## + +https://orgmode.org/worg/org-tutorials/org4beginners.html + +* `TAB/S-TAB`: Fold/Unfold +* `M-up/down`: move headline up/down +* `M-left/right`: promote or demote headline +* `M-RET`: add new headline + +## Treemacs ## + +https://github.com/Alexander-Miller/treemacs + +* `C-c C-w s`: Switch workspace + +## Lsp Mode ## + +### Python ### + +Install language LSP server: + +``` +brew install python-lsp-server +``` diff --git a/doc/mac.md b/doc/mac.md new file mode 100644 index 0000000..d124394 --- /dev/null +++ b/doc/mac.md @@ -0,0 +1,47 @@ +# Applications + +## Multimedia + +* [mplayer](http://www.mplayerhq.hu/design7/news.html) + +``` shell +brew install mplayer +``` + +## [Gnu Stow](https://www.gnu.org/software/stow/) + +Gnu Stow is a symlink manager that can be used to manager program configuration files or 'dotfiles'. + +``` shell +brew install stow +``` + +## [Amethyst](https://ianyh.com/amethyst/) + +Tiling window manager that doesn't need too heavy permissions. + +# Keyboard Shortcuts + +## Desktop + +* `CMD-m`: Minimize focused app. +* `CMD-q`: Kill the focused app. +* `CMD-TAB`: Iterate apps in dock with arrows. Hold `OPT` when releasing to open. +* `CTRL-CMD-q`: Lock Screen + +## Amethyst + +* `OPT-SHIFT-k`: Move focus ccw (j for cw) +* `OPT-SHIFT-p`: Move focus to ccw screen (n for cw) +* `CTRL-OPT-SHIFT-h`: Swap screens ccw (l for c) + +## Safari + +* Arrow keys: Scroll +* `OPT-Arrow`: Fast scroll +* `SHIFT-CMD-]`: Next or previous tab +* `CMD-t`: New tab +* `CMD-w`: Close active tab +* `CMD-[`: Previous or next page +* `CTRL-CMD-1`: Toggle bookmark view + diff --git a/doc/vm_notes.md b/doc/vm_notes.md new file mode 100644 index 0000000..f0848e1 --- /dev/null +++ b/doc/vm_notes.md @@ -0,0 +1,36 @@ +# Prepare Installation Images + +Want fully automated installation. + +## Deb + +Get `netinst` ISO: https://cdimage.debian.org/debian-cd/12.5.0/amd64/iso-cd/ + +Deb Preseeding: https://wiki.debian.org/DebianInstaller/Preseed + +ISO modification for preseeding: https://wiki.debian.org/DebianInstaller/Preseed/EditIso + + +# Set up KVM + +```sh +``` + +# Install Qemu + +```sh +sudo apt install qemu-utils qemu-system-x86 qemu-system-gui libvirt-daemon-system virtinst +``` + +Add user to libvirt group + +```sh +su -l +adduser $MY_USER libvirt +``` + +# References + +KVM and Debian: https://wiki.debian.org/KVM +Qemu and Debian: https://wiki.debian.org/QEMU +Qemu and libvrt: https://rabexc.org/posts/how-to-get-started-with-libvirt-on \ No newline at end of file diff --git a/basic_serverblock.txt b/infra/scripts/nginx/basic_serverblock.txt similarity index 100% rename from basic_serverblock.txt rename to infra/scripts/nginx/basic_serverblock.txt diff --git a/setup_basic_nginx.sh b/infra/scripts/nginx/setup_basic_nginx.sh similarity index 100% rename from setup_basic_nginx.sh rename to infra/scripts/nginx/setup_basic_nginx.sh diff --git a/setup_machine.sh b/infra/scripts/setup_machine.sh similarity index 100% rename from setup_machine.sh rename to infra/scripts/setup_machine.sh diff --git a/src/machine_admin/boot_image/__init__.py b/src/machine_admin/boot_image/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/machine_admin/boot_image/deb_preseed.py b/src/machine_admin/boot_image/deb_preseed.py new file mode 100644 index 0000000..b1161d9 --- /dev/null +++ b/src/machine_admin/boot_image/deb_preseed.py @@ -0,0 +1,74 @@ +from pathlib import Path +import logging + +from machine_admin.package_manager import SystemPackage +from machine_admin.util import run_op + +logger = logging.getLogger(__name__) + +class DebPreseeder: + + def __init__(self, iso_path: Path, preseed_path: Path): + self.iso_path = iso_path + self.preseed_path = preseed_path + + self.udevil = SystemPackage("udevil") + self.gzip = SystemPackage("gzip") + self.mount_point = None + + def mount_iso(self): + self.udevil.check_available() + op = f"udevil mount {self.iso_path}" + logger.info(f"Mounting iso | {op}") + output = run_op(op, capture_output=True) + self.mount_point = Path(output.split(" ")[3]) + + def add_preseed_file(self): + + # Copy ISO content to writeable location + op = f"cp -rT {self.mount_point} {self.workdir}/isofiles/" + run_op(op) + + # Make path writeable + install_arch_path = f"{self.workdir}/isofiles/install.amd" + run_op(f"chmod +w -R {install_arch_path}") + + # Extract initrd + self.gzip.check_available() + initrd_path = f"{install_arch_path}/initrd" + run_op(f"gunzip {initrd_path}.gz") + + # Add the preseed file + op = f"echo {self.preseed_path} | cpio -H newc -o -A -F {initrd_path}" + run_op(op) + + # Recompress the initrd + run_op(f"gzip {initrd_path}") + + # Restore path permissions + run_op(f"chmod -w -R {install_arch_path}") + + def regenerate_checksum(self): + checksum_path = f"{self.workdir}/isofiles/md5sum.txt" + run_op(f"chmod +w {checksum_path}") + + def regenerate_iso(self, workdir: Path): + + self.mount_iso() + + self.add_preseed_file() + + self.regenerate_checksum() + + + + + + + + + + + + + diff --git a/src/machine_admin/firewall.py b/src/machine_admin/firewall.py index ee5ba35..3180eb6 100644 --- a/src/machine_admin/firewall.py +++ b/src/machine_admin/firewall.py @@ -2,6 +2,8 @@ import logging from .util import run_op +logger = logging.getLogger(__name__) + class UfwInterface: def __init__(self): @@ -9,12 +11,12 @@ class UfwInterface: def enable(self): op = "ufw enable" - logging.info(f"Enabling ufw: {op}") + logger.info(f"Enabling ufw: {op}") run_op(op) def allow_app(self, app_name: str): op = f"ufw allow {app_name}" - logging.info(f"Allowing ufw app: {op}") + logger.info(f"Allowing ufw app: {op}") run_op(op) class Firewall: diff --git a/src/machine_admin/machine.py b/src/machine_admin/machine.py index 1a2a7e0..3001e40 100644 --- a/src/machine_admin/machine.py +++ b/src/machine_admin/machine.py @@ -21,7 +21,7 @@ class Machine: self.ssh_config.restart_service() def setup(self): - self.package_manager.update() + self.package_manager.upgrade() self.user_manager.setup_user(self.user) self.enable_firewall() self.secure_ssh_config() diff --git a/src/machine_admin/package_manager.py b/src/machine_admin/package_manager.py index e7d25ae..288518a 100644 --- a/src/machine_admin/package_manager.py +++ b/src/machine_admin/package_manager.py @@ -1,6 +1,9 @@ import logging -from .util import run_op +from .platform.distro import Distro +from .util import run_op, has_program + +logger = logging.getLogger(__name__) class AptInterface: @@ -9,20 +12,35 @@ class AptInterface: def update(self): op = "apt-get update" - logging.info(f"Updating apt: {op}") + logger.info(f"Updating apt: {op}") run_op(op) def upgrade(self): op = "apt-get -y upgrade" - logging.info(f"Upgrading via apt: {op}") + logger.info(f"Upgrading via apt: {op}") run_op(op) def install_packages(self, packages: list): - packages_str = "".join(packages) + packages_str = " ".join(packages) op = f"apt-get install -y {packages_str}" - logging.info(f"Installing packages: {op}") + logger.info(f"Installing packages: {op}") run_op(op) - + +class SystemPackage: + + def __init__(self, name): + self.name = name + self.package_alt_names = {} + + def check_available(self): + if has_program(self.name): + return + + msg = f"Program {self.name} not found" + if Distro.has_apt(): + msg = f"Program not found. Install with: sudo apt-get install {self.name}" + raise RuntimeError(msg) + class PackageManager: diff --git a/src/machine_admin/platform/__init__.py b/src/machine_admin/platform/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/machine_admin/platform/distro.py b/src/machine_admin/platform/distro.py new file mode 100644 index 0000000..63482bf --- /dev/null +++ b/src/machine_admin/platform/distro.py @@ -0,0 +1,11 @@ +from machine_admin.util import has_program + +class Distro: + + def __init__(self, name): + self.name = name + self.version = "" + + @staticmethod + def has_apt(self): + return has_program("apt-get") \ No newline at end of file diff --git a/src/machine_admin/platform/linux.py b/src/machine_admin/platform/linux.py new file mode 100644 index 0000000..0b3875c --- /dev/null +++ b/src/machine_admin/platform/linux.py @@ -0,0 +1,6 @@ +import sys + +from .distro import Distro + +def is_linux(): + return sys.platform == "linux" or sys.platform == "linux2" \ No newline at end of file diff --git a/src/machine_admin/ssh_config.py b/src/machine_admin/ssh_config.py index 6b65291..ad0b8d9 100644 --- a/src/machine_admin/ssh_config.py +++ b/src/machine_admin/ssh_config.py @@ -3,6 +3,8 @@ import logging from .util import run_op +logger = logging.getLogger(__name__) + class SshConfig: def __init__(self): @@ -13,14 +15,16 @@ class SshConfig: "UsePAM": "No"} def sync_target_values(self): - logging.info(f"Updating ssh config in: {self.config_path}") + logger.info(f"Updating ssh config in: {self.config_path}") pass def restart_service(self): op = "systemctl restart ssh" - logging.info(f"Restarting ssh service: {op}") + logger.info(f"Restarting ssh service: {op}") run_op(op) def copy_ssh_dir_to_user(self, username:str): op = f"rsync --archive --chown={username}:{username} ~/.ssh /home/{username}" + logger.info(f"Copying ssh dir to user: {op}") + run_op(op) \ No newline at end of file diff --git a/src/machine_admin/user.py b/src/machine_admin/user.py index c44d7c3..f51bd90 100644 --- a/src/machine_admin/user.py +++ b/src/machine_admin/user.py @@ -1,6 +1,8 @@ import logging from .util import run_op +logger = logging.getLogger(__name__) + class User: def __init__(self, name, has_sudo=False): self.name = name @@ -8,20 +10,20 @@ class User: class UserManager: - def __init__(): + def __init__(self): pass def setup_user(self, user: User): - add_user(user) + self.add_user(user) if user.has_sudo: - add_user_to_sudo(user) + self.add_user_to_sudo(user) def add_user(self, user: User): - op = f"adduser {user.name}" - logging.info(f"Adding user: {op}") + op = f'adduser {user.name} --disabled-password --comment ""' + logger.info(f"Adding user: {op}") run_op(op) def add_user_to_sudo(self, user: User): op = f"usermod -aG sudo {user.name}" - logging.info(f"Adding user to sudo: {op}") + logger.info(f"Adding user to sudo: {op}") run_op(op) \ No newline at end of file diff --git a/src/machine_admin/util.py b/src/machine_admin/util.py index 7ae006e..5d05154 100644 --- a/src/machine_admin/util.py +++ b/src/machine_admin/util.py @@ -1,13 +1,34 @@ import subprocess import logging +logger = logging.getLogger(__name__) + _DRY_RUN = False def set_is_dry_run(is_dry_run: bool): _DRY_RUN = is_dry_run -def run_op(op: str): - if _DRY_RUN: - return subprocess.run(op, shell=True) +def run_op(op: str, capture_output:bool = False, + cwd:str = None): + if not _DRY_RUN: + if capture_output: + ret = subprocess.run(op, + shell=True, + capture_output = True, + text = True, + cwd=cwd) + ret.check_returncode() + return ret.stdout + else: + ret = subprocess.run(op, shell=True, cwd=cwd) + ret.check_returncode() else: - logging.info(f"Dry Run | {op}") \ No newline at end of file + logger.info(f"Dry Run | {op}") + +def has_program(program_name: str): + op = f"which {program_name}" + ret = subprocess.run(op, + shell=True, + capture_output = True, + text = True) + return len(ret.stdout) \ No newline at end of file diff --git a/src/machine_setup.py b/src/machine_setup.py index 50bbbd4..f2ac28d 100644 --- a/src/machine_setup.py +++ b/src/machine_setup.py @@ -3,6 +3,9 @@ import logging from machine_admin.user import User from machine_admin.machine import Machine +from machine_admin.util import set_is_dry_run + +logger = logging.getLogger(__name__) if __name__ == "__main__": @@ -10,14 +13,18 @@ if __name__ == "__main__": prog='MachineSetup', description='Scripts for machine provisioning') - parser.add_argument('--username', + parser.add_argument('username', help="Name of the default non-root user") parser.add_argument('--dry_run', help="If set then don't change the system state - used for testing.", default = False) args = parser.parse_args() + set_is_dry_run(args.dry_run) + logging.basicConfig() + logging.getLogger().setLevel(logging.INFO) + user = User(args.username, has_sudo=True) machine = Machine(user) machine.setup() diff --git a/src/photos/convert_heic.sh b/src/photos/convert_heic.sh new file mode 100755 index 0000000..55c0d12 --- /dev/null +++ b/src/photos/convert_heic.sh @@ -0,0 +1,4 @@ +for f in $(find . -name '*.HEIC'); + do echo "Converting $f"; +/home/jgrogan/code/ImageMagick-7.1.1-35/utilities/magick mogrify -format png -quality 100% "$f"; +done; diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 0000000..2b0f393 --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,2 @@ +pytest +.pytest_cache \ No newline at end of file diff --git a/test/test_machine_setup.py b/test/test_machine_setup.py new file mode 100644 index 0000000..2bf988b --- /dev/null +++ b/test/test_machine_setup.py @@ -0,0 +1,11 @@ +from machine_admin.util import set_is_dry_run +from machine_admin.user import User +from machine_admin.machine import Machine + +set_is_dry_run(True) + +def test_machine_setup(): + + user = User("test_user", has_sudo=True) + machine = Machine(user) + machine.setup() \ No newline at end of file