From 5c01299d14eb0f963be3537f40de929707824774 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 15 Mar 2021 07:25:09 +0300 Subject: [PATCH 1/6] build --- .github/workflows/release-build-appimage.yml | 29 -------- .github/workflows/release-build-portable.yml | 29 -------- .github/workflows/release-python-publish.yml | 28 -------- .github/workflows/release.yml | 70 ++++++++++++++++++++ 4 files changed, 70 insertions(+), 86 deletions(-) delete mode 100644 .github/workflows/release-build-appimage.yml delete mode 100644 .github/workflows/release-build-portable.yml delete mode 100644 .github/workflows/release-python-publish.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release-build-appimage.yml b/.github/workflows/release-build-appimage.yml deleted file mode 100644 index b29dc99..0000000 --- a/.github/workflows/release-build-appimage.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Build xxh AppImage - -on: - release: - types: [created] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: Install xonsh shell - run: pip install xonsh - - name: Build xxh-appimage - run: xonsh xxh-appimage-build.xsh - - name: Test - run: ls -lah build/xxh-x86_64.AppImage - - name: Release - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') - with: - files: | - build/xxh-x86_64.AppImage - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-build-portable.yml b/.github/workflows/release-build-portable.yml deleted file mode 100644 index 024f439..0000000 --- a/.github/workflows/release-build-portable.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Build xxh portable - -on: - release: - types: [created] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: Install xonsh shell - run: pip install xonsh - - name: Build xxh-portable-musl-alpine-build - run: xonsh xxh-portable-musl-alpine-build.xsh - - name: Test - run: ls -lah build/xxh-portable-musl-alpine-Linux-x86_64.tar.gz - - name: Release - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') - with: - files: | - build/xxh-portable-musl-alpine-Linux-x86_64.tar.gz - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-python-publish.yml b/.github/workflows/release-python-publish.yml deleted file mode 100644 index 87ce4bb..0000000 --- a/.github/workflows/release-python-publish.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Upload Python Package - -on: - release: - types: [created] - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..eb551f5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,70 @@ +name: Release + +on: + release: + types: [created] + +jobs: + build-appimage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install xonsh shell + run: pip install xonsh + - name: Build xxh-appimage + run: xonsh xxh-appimage-build.xsh + - name: Test + run: ls -lah build/xxh-x86_64.AppImage + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: | + build/xxh-x86_64.AppImage + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + build-portable: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install xonsh shell + run: pip install xonsh + - name: Build xxh-portable-musl-alpine-build + run: xonsh xxh-portable-musl-alpine-build.xsh + - name: Test + run: ls -lah build/xxh-portable-musl-alpine-Linux-x86_64.tar.gz + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: | + build/xxh-portable-musl-alpine-Linux-x86_64.tar.gz + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + publish-python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* From 5897a0f0d6305b93ff75df4bd09365c2ea853331 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 15 Mar 2021 07:28:18 +0300 Subject: [PATCH 2/6] build --- .github/workflows/pullrequest-test.yml | 2 +- .github/workflows/push-test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pullrequest-test.yml b/.github/workflows/pullrequest-test.yml index 631003f..592cc3f 100644 --- a/.github/workflows/pullrequest-test.yml +++ b/.github/workflows/pullrequest-test.yml @@ -3,7 +3,7 @@ name: Test xxh on pull request on: pull_request jobs: - deploy: + test-full: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/push-test.yml b/.github/workflows/push-test.yml index 399cadb..c455ffd 100644 --- a/.github/workflows/push-test.yml +++ b/.github/workflows/push-test.yml @@ -3,7 +3,7 @@ name: Test xxh on push on: push jobs: - deploy: + test-short: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From 9c902a673367ad1e60d9c2684565878573a924ec Mon Sep 17 00:00:00 2001 From: a Date: Mon, 15 Mar 2021 07:44:03 +0300 Subject: [PATCH 3/6] Moved Python package --- setup.py | 7 ++++--- xxh => xxh/xxh | 0 {xxh_xxh => xxh/xxh_xxh}/__init__.py | 0 {xxh_xxh => xxh/xxh_xxh}/config.xxhc | 0 {xxh_xxh => xxh/xxh_xxh}/shell.py | 0 {xxh_xxh => xxh/xxh_xxh}/xxh.bash | 0 {xxh_xxh => xxh/xxh_xxh}/xxh.py | 0 {xxh_xxh => xxh/xxh_xxh}/xxh.xsh | 0 {xxh_xxh => xxh/xxh_xxh}/xxh.zsh | 0 9 files changed, 4 insertions(+), 3 deletions(-) rename xxh => xxh/xxh (100%) rename {xxh_xxh => xxh/xxh_xxh}/__init__.py (100%) rename {xxh_xxh => xxh/xxh_xxh}/config.xxhc (100%) rename {xxh_xxh => xxh/xxh_xxh}/shell.py (100%) rename {xxh_xxh => xxh/xxh_xxh}/xxh.bash (100%) rename {xxh_xxh => xxh/xxh_xxh}/xxh.py (100%) rename {xxh_xxh => xxh/xxh_xxh}/xxh.xsh (100%) rename {xxh_xxh => xxh/xxh_xxh}/xxh.zsh (100%) diff --git a/setup.py b/setup.py index 82c17b1..7668c4c 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ import setuptools -from xxh_xxh import __version__ +from xxh.xxh_xxh import __version__ with open("README.md", "r", encoding="utf8") as fh: long_description = fh.read() @@ -22,9 +22,10 @@ 'pyyaml' ], platforms='Unix-like', - scripts=['xxh', 'xxh_xxh/xxh.zsh', 'xxh_xxh/xxh.xsh', 'xxh_xxh/xxh.bash'], + scripts=['xxh/xxh', 'xxh/xxh_xxh/xxh.zsh', 'xxh/xxh_xxh/xxh.xsh', 'xxh/xxh_xxh/xxh.bash'], package_data={'xxh_xxh': ['*.py', '*.xxhc', 'xxh.*']}, - packages=setuptools.find_packages(), + packages=['xxh_xxh'], + package_dir={'xxh_xxh': 'xxh/xxh_xxh'}, classifiers=[ "Programming Language :: Python :: 3", "Programming Language :: Unix Shell", diff --git a/xxh b/xxh/xxh similarity index 100% rename from xxh rename to xxh/xxh diff --git a/xxh_xxh/__init__.py b/xxh/xxh_xxh/__init__.py similarity index 100% rename from xxh_xxh/__init__.py rename to xxh/xxh_xxh/__init__.py diff --git a/xxh_xxh/config.xxhc b/xxh/xxh_xxh/config.xxhc similarity index 100% rename from xxh_xxh/config.xxhc rename to xxh/xxh_xxh/config.xxhc diff --git a/xxh_xxh/shell.py b/xxh/xxh_xxh/shell.py similarity index 100% rename from xxh_xxh/shell.py rename to xxh/xxh_xxh/shell.py diff --git a/xxh_xxh/xxh.bash b/xxh/xxh_xxh/xxh.bash similarity index 100% rename from xxh_xxh/xxh.bash rename to xxh/xxh_xxh/xxh.bash diff --git a/xxh_xxh/xxh.py b/xxh/xxh_xxh/xxh.py similarity index 100% rename from xxh_xxh/xxh.py rename to xxh/xxh_xxh/xxh.py diff --git a/xxh_xxh/xxh.xsh b/xxh/xxh_xxh/xxh.xsh similarity index 100% rename from xxh_xxh/xxh.xsh rename to xxh/xxh_xxh/xxh.xsh diff --git a/xxh_xxh/xxh.zsh b/xxh/xxh_xxh/xxh.zsh similarity index 100% rename from xxh_xxh/xxh.zsh rename to xxh/xxh_xxh/xxh.zsh From a1ffa09e58f860021b743643b5ebb11527a99ee0 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 15 Mar 2021 07:44:34 +0300 Subject: [PATCH 4/6] Bump 0.8.8 --- xxh/xxh_xxh/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xxh/xxh_xxh/__init__.py b/xxh/xxh_xxh/__init__.py index 8843c51..a02276f 100644 --- a/xxh/xxh_xxh/__init__.py +++ b/xxh/xxh_xxh/__init__.py @@ -1 +1 @@ -__version__ = '0.8.7' +__version__ = '0.8.8' From d4658fdbd11f7710f7d946cd009ed3a99880a569 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 15 Mar 2021 08:14:10 +0300 Subject: [PATCH 5/6] Moving xxh development environment from xxh/xxh-dev to xxh/xxh/xde --- .github/workflows/pullrequest-test.yml | 16 +- .github/workflows/release.yml | 2 +- README.md | 2 +- xde/.gitattributes | 3 + xde/.gitignore | 31 ++ xde/LICENSE | 25 ++ xde/README.md | 94 ++++++ xde/docker-compose.yml | 60 ++++ xde/keys/README.md | 6 + xde/keys/id_rsa | 38 +++ xde/keys/id_rsa.pub | 1 + xde/keys/keys_password/README.md | 6 + xde/keys/keys_password/id_rsa | 30 ++ xde/keys/keys_password/id_rsa.pub | 1 + xde/requirements.txt | 9 + xde/tests/bash/test_env.sh | 1 + xde/tests/bash/test_seamless.sh | 1 + xde/tests/tests.xsh | 307 +++++++++++++++++++ xde/tests/xonsh/test_env.py | 18 ++ xde/tests/xonsh/test_pip_package_import.xsh | 4 + xde/tests/xonsh/test_pip_package_install.xsh | 3 + xde/tests/xonsh/test_pip_upgrade.xsh | 3 + xde/tests/xonsh/test_plugins.xsh | 11 + xde/tests/xonsh/test_python.xsh | 3 + xde/tests/xonsh/test_xonsh_run_xonsh.xsh | 1 + xde/tests/xonsh/test_xontrib.xsh | 9 + xde/tests/xonsh/xonsh_script.xsh | 5 + xde/tests/zsh/test_env.zsh | 1 + xde/xde | 175 +++++++++++ xde/xde-test-fast | 11 + xde/xxh-dev-centos-k.Dockerfile | 6 + xde/xxh-dev-start-entrypoint.sh | 14 + xde/xxh-dev-start.Dockerfile | 24 ++ xde/xxh-dev-start.sh | 53 ++++ xde/xxh-dev-ubuntu-k.Dockerfile | 15 + xde/xxh-dev-ubuntu-kf.Dockerfile | 6 + requirements.txt => xxh/requirements.txt | 0 xxh/xxh | 1 - 38 files changed, 982 insertions(+), 14 deletions(-) create mode 100644 xde/.gitattributes create mode 100644 xde/.gitignore create mode 100644 xde/LICENSE create mode 100644 xde/README.md create mode 100644 xde/docker-compose.yml create mode 100644 xde/keys/README.md create mode 100644 xde/keys/id_rsa create mode 100644 xde/keys/id_rsa.pub create mode 100644 xde/keys/keys_password/README.md create mode 100644 xde/keys/keys_password/id_rsa create mode 100644 xde/keys/keys_password/id_rsa.pub create mode 100644 xde/requirements.txt create mode 100644 xde/tests/bash/test_env.sh create mode 100644 xde/tests/bash/test_seamless.sh create mode 100755 xde/tests/tests.xsh create mode 100644 xde/tests/xonsh/test_env.py create mode 100644 xde/tests/xonsh/test_pip_package_import.xsh create mode 100644 xde/tests/xonsh/test_pip_package_install.xsh create mode 100644 xde/tests/xonsh/test_pip_upgrade.xsh create mode 100644 xde/tests/xonsh/test_plugins.xsh create mode 100644 xde/tests/xonsh/test_python.xsh create mode 100644 xde/tests/xonsh/test_xonsh_run_xonsh.xsh create mode 100644 xde/tests/xonsh/test_xontrib.xsh create mode 100644 xde/tests/xonsh/xonsh_script.xsh create mode 100644 xde/tests/zsh/test_env.zsh create mode 100755 xde/xde create mode 100755 xde/xde-test-fast create mode 100644 xde/xxh-dev-centos-k.Dockerfile create mode 100755 xde/xxh-dev-start-entrypoint.sh create mode 100644 xde/xxh-dev-start.Dockerfile create mode 100755 xde/xxh-dev-start.sh create mode 100644 xde/xxh-dev-ubuntu-k.Dockerfile create mode 100644 xde/xxh-dev-ubuntu-kf.Dockerfile rename requirements.txt => xxh/requirements.txt (100%) diff --git a/.github/workflows/pullrequest-test.yml b/.github/workflows/pullrequest-test.yml index 592cc3f..74a1ac8 100644 --- a/.github/workflows/pullrequest-test.yml +++ b/.github/workflows/pullrequest-test.yml @@ -11,20 +11,14 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.8' - - name: Install xonsh - run: pip install xonsh - - name: Install xxh-dev - run: | - cd .. - git clone --depth 1 https://github.com/xxh/xxh-dev - cd xxh-dev - pip install -r requirements.txt + - name: Install xde + run: cd xde && pip install -r requirements.txt - name: xde build - run: cd ../xxh-dev && ./xde build + run: cd xde && ./xde build - name: xde up - run: cd ../xxh-dev && ./xde up + run: cd xde && ./xde up - name: xde test - run: cd ../xxh-dev && ./xde test --not-interactive + run: cd xde && ./xde test --not-interactive diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eb551f5..57436f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,7 +49,7 @@ jobs: build/xxh-portable-musl-alpine-Linux-x86_64.tar.gz env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - publish-python: + publish-python-package: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index ed44924..70f192d 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ platforms that could be forked and stuffed. - [New questions](https://github.com/xxh/xxh/wiki#new-questions) ## Development -In the [xxh-dev](https://github.com/xxh/xxh-dev) repo there is full [dockerised](https://www.docker.com/) environment +In the [xxh development environment](https://github.com/xxh/xxh/tree/master/xde) repo there is full [dockerised](https://www.docker.com/) environment for development, testing and contribution. The process of testing and development is orchestrated by `xde` tool and is as easy as possible. diff --git a/xde/.gitattributes b/xde/.gitattributes new file mode 100644 index 0000000..5b9c3e7 --- /dev/null +++ b/xde/.gitattributes @@ -0,0 +1,3 @@ +*.xsh text linguist-language=Python +xde text linguist-language=Python +xde-test-fast text linguist-language=Python diff --git a/xde/.gitignore b/xde/.gitignore new file mode 100644 index 0000000..dad05d1 --- /dev/null +++ b/xde/.gitignore @@ -0,0 +1,31 @@ +xxh +squashfs-root +appimage +xxh-shell* +xxh-plugin* +*.pyc +*.out +*.xcf +*.egg +.eggs/ +.pytest_cache/ +build/ +dist/ +xxh_xxh.egg-info/ +venv/ + +# temporary files from vim and emacs +*~ +*# +.#* +*.swp +*.swo + +# Editor project files +*.komodo* +.cache +.idea +.vscode/ + +# Mac +.DS_Store diff --git a/xde/LICENSE b/xde/LICENSE new file mode 100644 index 0000000..e63e627 --- /dev/null +++ b/xde/LICENSE @@ -0,0 +1,25 @@ +BSD 2-Clause License + +Copyright (c) 2020, xxh +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/xde/README.md b/xde/README.md new file mode 100644 index 0000000..3e7fe03 --- /dev/null +++ b/xde/README.md @@ -0,0 +1,94 @@ +Development and test environment for [xxh/xxh](https://github.com/xxh/xxh) contains +network of docker containers which allow to test the ssh connections and xxh functionality +with or without [AppImage FUSE](https://github.com/AppImage/AppImageKit/wiki/FUSE). + +Docker containers in the network: + +| Hostname | Auth | FUSE | rsync | users | +|-----------|------------------|------|-------|----------------------------------| +| start | `./xde g start` | | | `root`, `uxonsh`, `uzsh`, `ufish`, `ubash` | +| ubuntu_k | key | | | `root` | +| ubuntu_kf | key | yes | yes | `root` | +| centos_k | key | | | `root` | +| arch_p | password | | | `root`, `docker` | + +Every container has `/xxh` it is the volume. For example if you'll add a file to `tests/new.xsh` +it appears on all hosts immediately in `/xxh/xde/tests/new.xsh`. + +## Workflow + +This workflow was originally developed on `ubuntu 20.04`, `docker 19.03.5`, `docker-compose 1.25.3`, `xonsh 0.9.14`, `pycharm 2019.3.3`. + +0. Install packages: +```bash +apt install -y docker git # Or install docker from official repo - https://docs.docker.com/engine/install/ubuntu/ +pip install -r requirements.txt +``` + +1. Create directory for xxh development: +```bash +mkdir -p ~/git && cd ~/git +git clone https://github.com/xxh/xxh +cd xxh/xde +./xde build # build docker containers +./xde up # run docker containers +./xde test # run tests first time +``` + +2. Open `~/git/xxh` in your IDE to make changes and commit many repos. +3. Now you can go to `start` host and try your first connect using xxh: +```shell script +./xde goto start + +# Press UP key to get connection strings to other hosts from bash history. +# Here xxh will be from /xxh/xxh/ that is your local directory +#root@start> +xxh -i ~/id_rsa root@ubuntu_k + +#root@start> #Try from another shell: +su uzsh +#uzsh@start> +xxh +I xxh-plugin-zsh-ohmyzsh +#uzsh@start> +xxh -i ~/id_rsa root@ubuntu_k +#root@ubuntu_k% +echo $ZSH_THEME && exit +#agnoster +#uzsh@start> +source xxh.zsh -i ~/id_rsa root@ubuntu_k +#root@ubuntu_k% +echo $ZSH_THEME +#bira +``` + +4. Change the code in IDE and run `xxh` on `start` container. It's so easy! +5. Run tests `./xde t` (don't forget about `./xde t --help` and fast mode `./xde t -sr`) +6. After end of work you can `./xde stop` or `./xde remove` the containers. +7. You rock! Try to find easter egg in the xxh code now. + +## xxh development environment tool + +```shell script +./xde -h +#usage: xde +# +#xxh development environment commands: +# +# clone Git clone repos from https://github.com/xxh +# build Build the docker containers and get the xxh code if ./xxh is not exists +# up Docker-compose up the containers +# test t Run tests +# goto g Open bash by the container name part +# start Docker-compose start the containers +# stop Docker-compose stop the containers +# remove Docker-compose remove the containers +# +#Try `./xde --help` to get more info. +# +#positional arguments: +# command Command to run +# +#optional arguments: +# -h, --help show this help message and exit + +``` diff --git a/xde/docker-compose.yml b/xde/docker-compose.yml new file mode 100644 index 0000000..ee59624 --- /dev/null +++ b/xde/docker-compose.yml @@ -0,0 +1,60 @@ +version: "3" +services: + start: + hostname: start + image: xxh/xxh-dev-start + depends_on: + - ubuntu_k + - ubuntu_kf + - arch_p + - centos_k + volumes: + - ..:/xxh + working_dir: /root + + ubuntu_k: + hostname: ubuntu_k + image: xxh/xxh-dev-ubuntu-k + volumes: + - ..:/xxh + + ubuntu_kf: + hostname: ubuntu_kf + image: xxh/xxh-dev-ubuntu-kf + volumes: + - ..:/xxh + cap_add: + - SYS_ADMIN + - MKNOD + devices: + - /dev/fuse + security_opt: + - apparmor:unconfined + + centos_k: + hostname: centos_k + image: xxh/xxh-dev-centos-k + volumes: + - ..:/xxh + + arch_p: + hostname: arch_p + image: yantis/archlinux-small-ssh-hpn # https://github.com/yantis/docker-archlinux-ssh-hpn + volumes: + - ..:/xxh + +# # Alpine is not supported (https://github.com/AppImage/AppImageKit/issues/1015) +# alpine_p: +# hostname: alpine_without_fuse +# image: linuxserver/openssh-server +# environment: +# - PUID=1000 +# - PGID=1000 +# - TZ=Europe/London +# - USER_NAME=luke +# - USER_PASSWORD=skywalker +# - PASSWORD_ACCESS=true +# - SUDO_ACCESS=true +# - PUBLIC_KEY_FILE=/xxh-dev/keys/id_rsa.pub +# volumes: +# - ..:/xxh \ No newline at end of file diff --git a/xde/keys/README.md b/xde/keys/README.md new file mode 100644 index 0000000..013ca03 --- /dev/null +++ b/xde/keys/README.md @@ -0,0 +1,6 @@ +Keys without password. + +Regenerate keys: +``` +ssh-keygen -t rsa -f id_rsa +``` \ No newline at end of file diff --git a/xde/keys/id_rsa b/xde/keys/id_rsa new file mode 100644 index 0000000..6f982d2 --- /dev/null +++ b/xde/keys/id_rsa @@ -0,0 +1,38 @@ +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn +NhAAAAAwEAAQAAAYEAvH44coH05sR914EaBAg4s3hH03zct10XnrdHb/EVoaQ4IBO0Ht3f +rIIofLOqJJYnaRjAKYrDA04wOCdaI0awdj9bO4tk47lc+3g8cHQEoeyAeM2mPoKzFPoEYq +umIRrq2Fz/j6Prft/DEEV1Z4fQoVOkYe07aBPhnIktcBnfkJEp15Kw1Yc0gOWT0+zick5V +FypUPDH+4VM8xBHsBAh9JH4B0L0Pnn+wpfdhcPppn7JhmQQTeATHtAgOIYwNOZLjVpo8xL +rwA2FzDBvRnzaGP8tKgPbebqasAZIr/7CKYD6N7IFBzaEMsJu0JXyVww+XXF9XAZ1zeRY2 +PHX0nUq0mAc0eTfISuMnXkCvxPcRcZ6cJfn62+UqLWQDUU5BRY3uQkf7ZGdyynF4PQIeIG +R2Kg/VDEQ+56nY7DY765lCQ+NEO2HyKsVcjUfz5Aa5FvjNT9Ib4xoJjyG4zUP8GJyLciDG +Rwa1RhKzJRqY624PGvD4tPtnINquRQ7+YmkzidyVAAAFgJaba5iWm2uYAAAAB3NzaC1yc2 +EAAAGBALx+OHKB9ObEfdeBGgQIOLN4R9N83LddF563R2/xFaGkOCATtB7d36yCKHyzqiSW +J2kYwCmKwwNOMDgnWiNGsHY/WzuLZOO5XPt4PHB0BKHsgHjNpj6CsxT6BGKrpiEa6thc/4 ++j637fwxBFdWeH0KFTpGHtO2gT4ZyJLXAZ35CRKdeSsNWHNIDlk9Ps4nJOVRcqVDwx/uFT +PMQR7AQIfSR+AdC9D55/sKX3YXD6aZ+yYZkEE3gEx7QIDiGMDTmS41aaPMS68ANhcwwb0Z +82hj/LSoD23m6mrAGSK/+wimA+jeyBQc2hDLCbtCV8lcMPl1xfVwGdc3kWNjx19J1KtJgH +NHk3yErjJ15Ar8T3EXGenCX5+tvlKi1kA1FOQUWN7kJH+2RncspxeD0CHiBkdioP1QxEPu +ep2Ow2O+uZQkPjRDth8irFXI1H8+QGuRb4zU/SG+MaCY8huM1D/Bici3IgxkcGtUYSsyUa +mOtuDxrw+LT7ZyDarkUO/mJpM4nclQAAAAMBAAEAAAGAZvFIQsFWzvFhCOEpJCqZTOrTz1 +zR7hKZY8UU7r3r4mude2dHZ9Qk5EHxeqlksraOr4ck64stFhENpJ1ZTQq+NNZFLS/j2brS +XUQ9Pe0e2pAfWdp2gI1A4KPq4Th9cncQ6zlfGUOcyZ3LGofUpsY12uWjURgO6bhx7tu8RQ +zcXEfrYiBrEzLUKdE3+vXUdjjxn8UuW/GyEb0z4qEnFMzOC7AFrcqTTSzPfNSuEtWv+dei +Fb90Iof5yQjdLtUeI+Nj9TmTVzQg++pCTJPq7DW5niJQ/E1T2N8GlYYVidFCphK3ILlM60 +4Qy99UyOwiBpHaU121auZpsvYrN7fmxgJxe1NxKlExWaLb3oRJ8Oq5T6632OezWn9iXJjn +aB5znktS6QrtRV02Ptal+FZgRvs7/+/xaTk9TM65uLkTHQCsDvPYolGwYZIpAb6wd8+a+h +Hy4SOqxBRWBDs6JTqdUgMePv4ZWBZMGf3XQ9x6JGIDavb/7RaVkOOUKJHtNei3/N3BAAAA +wQCUKU4KopXaPqLspfrPDD1fgSo1LOAsb4sDFgP6gzJnbdIaWO0a0MNyzC/Szw9CULrrIq +SrENEkV+z/g2FhliH3l8v5UabwP3OpE78Yt9o1Es3jxR4nI1PoSb5lZkMljXK+BjPz8uIt +BzcQ0WI7Urjv6iXia1B5qQkeA6lNfd2PCK+QVBU0lyvlJtsff0os5F551uel7qzxLuuuTH +A5AzLG8x2gq8aLNY86Mbc01PQVow/dRk+7M/xamJuhVZR+AucAAADBAORQ3TFmMAA52FET +9FQwuaBEoK/rLUfNAZbfh2JuEjq0YNNZ7ne7kZHKDZiGoDlF5lAUdQYJAWFbqatkpXvazI +GEyFDuWZSXN+R/z8nPCBB2gDea1dl5dG7L2BstvW0lFI6qODPtVPXGVxaNWAWw5hbxF1u7 +4JIMgps0FT2n2ltvHchf4TrmBnjWHplDms4j/6kHDgRRMcMMRaEtndXifyRyMNH8owDnK2 +Yfe+UusZSzr+A3SdgZ9fw2nXI08jQEgwAAAMEA01k4FqdRAZYqTXfYtu8ZRQisHYDDEwgh +vn2xbqJk+53hM1tpgPYaO+KygffTh9CNMPhPNdvv6JSuzI23Dl/U4M1ntdS8VrXAi7gd+E +MV5LQ+NVO6ON/GBgnHLzgBaH7tsnzCRZLAl1n1NoD1Cw4tWS+Nydyp0icTM7Eod46x1poW +9LVCH1iuK4GcI/eC6+3F2Pwtg7KR0E8jhPjwbGEgi0a6YHFhOySKXa5Wz1gkjiz0zb0dWF +2UYNV58cOMQL8HAAAABXBjQHBjAQIDBAU= +-----END OPENSSH PRIVATE KEY----- diff --git a/xde/keys/id_rsa.pub b/xde/keys/id_rsa.pub new file mode 100644 index 0000000..8e5aaab --- /dev/null +++ b/xde/keys/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC8fjhygfTmxH3XgRoECDizeEfTfNy3XReet0dv8RWhpDggE7Qe3d+sgih8s6oklidpGMApisMDTjA4J1ojRrB2P1s7i2TjuVz7eDxwdASh7IB4zaY+grMU+gRiq6YhGurYXP+Po+t+38MQRXVnh9ChU6Rh7TtoE+GciS1wGd+QkSnXkrDVhzSA5ZPT7OJyTlUXKlQ8Mf7hUzzEEewECH0kfgHQvQ+ef7Cl92Fw+mmfsmGZBBN4BMe0CA4hjA05kuNWmjzEuvADYXMMG9GfNoY/y0qA9t5upqwBkiv/sIpgPo3sgUHNoQywm7QlfJXDD5dcX1cBnXN5FjY8dfSdSrSYBzR5N8hK4ydeQK/E9xFxnpwl+frb5SotZANRTkFFje5CR/tkZ3LKcXg9Ah4gZHYqD9UMRD7nqdjsNjvrmUJD40Q7YfIqxVyNR/PkBrkW+M1P0hvjGgmPIbjNQ/wYnItyIMZHBrVGErMlGpjrbg8a8Pi0+2cg2q5FDv5iaTOJ3JU= pc@pc diff --git a/xde/keys/keys_password/README.md b/xde/keys/keys_password/README.md new file mode 100644 index 0000000..ac1810e --- /dev/null +++ b/xde/keys/keys_password/README.md @@ -0,0 +1,6 @@ +Keys with password. Password: `snail` + +Regenerate keys: +``` +ssh-keygen -t rsa -f id_rsa +``` \ No newline at end of file diff --git a/xde/keys/keys_password/id_rsa b/xde/keys/keys_password/id_rsa new file mode 100644 index 0000000..a59943f --- /dev/null +++ b/xde/keys/keys_password/id_rsa @@ -0,0 +1,30 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,687980AF32438B332AB5EF8DA707B8F9 + +7yYL10CzTNp7AxE66nuZauZ9Md4FmLq2RG/FN4Ztu2qL9j9X4ZufwdT1g/uMmHJq +WJt5o4RjxtSRVp/T6T5w0vHjPGU6tlSHKIIjzzq2UFkFVkCzbE6DBjpWOAaQGlW3 +PaN/MElZa2LCq2tm+8/qQijg5RhUYMTpCepwL8iZY29VVEtdiKTT5uiwsnxqqmsu +zDT1DbbEUhTrNH1x8CE0H1qJIw2YoIINbPKhfMhBgTG9e5P1bLRZAFHBjbZP53PG +17BvM6HI5N/LxuBYwAk3DyWz72M1wHriNKDjoTTzVa9hEAkfh4cf5J4YZLtGA8uz +dtPi2TCTnkvk0THBHgNuBeaFK7nUZsSEt9dWxCXo0nXxADeIy4gasW+nfLMxym8t +UtZLPtIZbidnJDoh529SNGuT3fgiTLWTCpI8O7ajS11CUrY2z+sEBX959SljlECj +V5/pXCpB/13Ijljwi8B8YVLrE8yyAqQqgFp0u/v6Nf2qrso4qCyZ3kB0Zk1KNKrj ++Rgo7/yCdkhMyy6lT9gLkxXGMF8iv/bxpi6pjUpQO9E3ULu8V+7xpbXoLbwecRDp +fO7vCTWiEy95mH+JoAMld1/zpZgXLlGRbe3wWv0wDt33kcNHkoUZNYVjUbsBGXbC +grCgyR0FgfXVuF1TUuSlqJTxQ0sBfA3sZixLhzyieDPhYKZNnHNW8sukPQZnzw96 +PhwF4yS+o3T5ZzfhgnEQ0km1QOAcVkyOJ+GsR/4Be/5Lae3h+qzhzpnbl7ebZH9S +98gXZQHvmJlr34jZQrtolqgzN6aELvxxD6tBxKFbISr0R3r4VCJAgrOVU5wkA5AT +RJBkgRTz/xIAbVAk2nGMddHlT0mMTuCMGoD8lCs5sfSDlhzSv2u6u3Orc8b8RNvh +EgG112S0KO66OlecJMviQnY0LeTQfWR3fe324Jg21QA/vyLha8uRBLOjnPM4Bgtv +7jJJYcdeWsLHsPqFUv2MtMqRXCgDhnLm2lOPDl3VQ+ew7kLU9A5e7noHeP5z//nk +0LP/etZuvY3TgjDIwBxgf4/AfHeIxx2srxc0lfaivLDI986z4YJ688NPyjsfTMjP +vZ6j+IwcB7Ewm1nLa+GQ1CJbm8ajpU8Jic3aCHirAkzpt4NZ3D2T2/ZKC69RY19E +ZbXC15Fw5kIEx4SASOln8qyJ6kp9NZDiZletJMByX2JAkWcwIUR2OXOwGq1PoSk4 +zpyfECm469TN6wjxvRz+RqWPGfHvSSlSbiM2J4upGjhpc64KBydlOCrZ6SFQnuLM +uYY2yAWXpnvWF2k6deHw7UDfB3/Rps6TgGAyt4O7W/a0OV1fbjbM5/UmZnIjeL7n +fGlT/wW6reRtbJ+8DPYpQ/zHMS5SDZFqJfGG0XL4i9Uz8EUsnp5yx2wLXj8pjzx8 +fD/wUZ6+iZEAl7gSB9KO7IU+xAMtPU4iqwZ0Ya6KtY9b0EE6Nn/Y90TaEmYvbpkP +0S2tTxTiKQX/EWsyrSOLWE16zEAz1EBQHJvcSPhgwhpevg5lUngaoVevwVjdxpfY +pJKemv9kG1S7As6k/khoSkff5gy1C+e+RwSiY7r8hDvtbQaCO6ucB8to6oHVNaA1 +-----END RSA PRIVATE KEY----- diff --git a/xde/keys/keys_password/id_rsa.pub b/xde/keys/keys_password/id_rsa.pub new file mode 100644 index 0000000..54e0583 --- /dev/null +++ b/xde/keys/keys_password/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDFLZkeEaATTj2ROUjdDFwYzkDU4DQhrHbYpgPJLjez+T8CnDmQpfRqWkZWvC8guf5rruDPEURHp+/S7qzbLhwOQxWoSTYooYJKVWD5KchNOXNUBbtx9zeG/m2ZxwbjgN2gBfmTAcYVkVhTCEqmz7ywGiC+ew8MI5mChISa42M28mxRi1DQIbvqUElWNm884JIZCrNx1gRN8HqOL7VMeFmxeRzLLc89GBul2mHypiG9IJd2YaPi8QA/6HXYyOTiGW6vA3xOtDW0Mfj2QUJO6+b/Q8zgWJuxGhq2VKbB3vyTEf8olFZKZcubbJe1VLKQ/+K7HVDLakdIKASGPryu7fdx root@start diff --git a/xde/requirements.txt b/xde/requirements.txt new file mode 100644 index 0000000..15faa7e --- /dev/null +++ b/xde/requirements.txt @@ -0,0 +1,9 @@ +docker-compose +argparse +pyyaml +setuptools +wheel +twine +xonsh +argcomplete +xontrib-argcomplete diff --git a/xde/tests/bash/test_env.sh b/xde/tests/bash/test_env.sh new file mode 100644 index 0000000..a45eba2 --- /dev/null +++ b/xde/tests/bash/test_env.sh @@ -0,0 +1 @@ +echo "test bash xxh $(basename -- "$(readlink -f $XXH_HOME)") and env=$TESTENV" diff --git a/xde/tests/bash/test_seamless.sh b/xde/tests/bash/test_seamless.sh new file mode 100644 index 0000000..f208e31 --- /dev/null +++ b/xde/tests/bash/test_seamless.sh @@ -0,0 +1 @@ +echo $OSH_THEME diff --git a/xde/tests/tests.xsh b/xde/tests/tests.xsh new file mode 100755 index 0000000..7be6374 --- /dev/null +++ b/xde/tests/tests.xsh @@ -0,0 +1,307 @@ +#!/usr/bin/env xonsh + +import sys, os, argparse, re, subprocess + +verbose = False +vverbose = False +not_interactive = False + +def SC(cmd): + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + [out, err] = proc.communicate() + return (out, err, proc) + +def cmd_str(c): + return c + +def check(name, cmd, expected_result): + try_count = 3 + while try_count > 0: + try_count -= 1 + print('TEST: '+name, end='...') + cmd = cmd.strip() + + if vverbose: + print(f'\nRUN: {cmd}', end=' ...') + + if 'xxh local' in cmd: + o, e, p = SC(cmd) + cmd_result = o.decode().strip() + else: + cmd_result = $(bash -c @(cmd)).strip() + cmd_result = re.sub('\x1b]0;.*\x07','', cmd_result) + cmd_result = re.sub(r'\x1b\[\d+m','', cmd_result) + cmd_result = re.sub(r'\x1b\[\d+;\d+;\d+m','', cmd_result) + + # https://github.com/xonsh/xonsh/issues/3555 and https://github.com/xonsh/xonsh/issues/3557 + cmd_result = re.sub(r'bash: cannot set terminal process group \(.*?\): Inappropriate ioctl for device\nbash: no job control in this shell\n', '', cmd_result) + + expected_result = expected_result.strip() + if cmd_result != expected_result or vverbose: + print('\n',end='') + if verbose: + print(f'CMD: {cmd}') + print(f"OUTPUT {repr(cmd_result)}\nEXPECT {repr(expected_result)} ") + + if cmd_result != expected_result: + if try_count > 0 or 'Bad file descriptor' in cmd_result: + print('RETRY') + continue + else: + print('ERROR!') + cmdv = cmd + ' +v # or with +vv' + if not not_interactive: + yn = input(f'Run verbose? [Y/n]: %s' % cmdv) + if yn.lower().strip() in ['y','']: + bash -c @(cmdv) + + if try_count > 0: + continue + else: + print('To debug run `./xde g start` and then: %s' % cmdv) + sys.exit(1) + + print('OK') + break + +if __name__ == '__main__': + + argp = argparse.ArgumentParser(description=f"xde test") + argp.add_argument('-r', '--remove', default=False, action='store_true', help="Remove xxh home before tests.") + argp.add_argument('-ni', '--not-interactive', default=False, action='store_true', help="Not interactive mode.") + argp.add_argument('-v', '--verbose', default=False, action='store_true', help="Verbose mode.") + argp.add_argument('-vv', '--vverbose', default=False, action='store_true', help="Super verbose mode.") + argp.add_argument('-s', '--shell', help="Shells to test") + argp.add_argument('-H', '--hosts', default=[], help="Comma separated hosts list") + argp.add_argument('-sr', '--skip-repos-update', default=False, action='store_true', help="Skip repos update before test running.") + argp.usage = argp.format_usage().replace('usage: tests.xsh', 'xde test') + opt = argp.parse_args() + verbose = opt.verbose + not_interactive = opt.not_interactive + if opt.vverbose: + verbose = vverbose = True + + git_verbose_arg = [] if verbose else ['-q'] + + if opt.hosts: + opt.hosts = opt.hosts.split(',') + + xxh = '/xxh/xxh/xxh' + local_xxh_home = '/root/local_xxh_home' + xxh_args = ['++local-xxh-home', local_xxh_home] + + hosts = {} + hosts['local'] = { + 'xxh_auth': [], + 'home': '/root' + } + hosts['ubuntu_k'] = { + 'user':'root', + 'home':'/root', + 'ssh_auth': ['-i', '/xxh/xde/keys/id_rsa'], + 'xxh_auth': ['-i', '/xxh/xde/keys/id_rsa'], + 'sshpass': [] + } + hosts['ubuntu_kf'] = hosts['ubuntu_k'] + hosts['centos_k'] = hosts['ubuntu_k'] + hosts['arch_p'] = { + 'user':'docker', + 'home':'/home/docker', + 'ssh_auth':[], + 'xxh_auth':['+P','docker'], + 'sshpass': ['sshpass', '-p', 'docker'] + } + + rm -rf /root/.ssh/known_hosts + + if opt.remove: + print(f'Remove start:{local_xxh_home}') + rm -rf @(local_xxh_home) + + if not pf'{local_xxh_home}/.xxh/shells'.exists(): + print("Don't forget `pip uninstall xxh-xxh` before tests") + print('First time of executing tests takes time because of downloading files. Take a gulp of water or a few :)') + + mkdir -p @(local_xxh_home)/.xxh/plugins @(local_xxh_home)/.xxh/shells + + print('Prepare repos (to avoid full update use --skip-repos-update)') + + xxh_shell_repos = {} + xxh_shell_repos['xxh-shell-xonsh'] = { + 'shells': ['xxh-shell-xonsh', 'xxh-shell-zsh'], + 'plugins': ['xxh-plugin-xonsh-pipe-liner', 'xxh-plugin-xonsh-theme-bar', 'xxh-plugin-xonsh-autojump'] + } + xxh_shell_repos['xxh-shell-zsh'] = { + 'shells': ['xxh-shell-zsh'], + 'plugins': ['xxh-plugin-zsh-ohmyzsh'] + } + xxh_shell_repos['xxh-shell-bash'] = { + 'shells': ['xxh-shell-bash'], + 'plugins': ['xxh-plugin-bash-ohmybash'] + } + xxh_shell_repos['xxh-shell-fish'] = { + 'shells': ['xxh-shell-fish'], + 'plugins': [] + } + + if opt.shell: + xxh_shell_repos = {opt.shell: xxh_shell_repos[opt.shell]} + + if not opt.skip_repos_update: + print(f'Remove {local_xxh_home}/.xxh/shells {local_xxh_home}/.xxh/plugins') + rm -rf @(local_xxh_home)/.xxh/shells @(local_xxh_home)/.xxh/plugins + + for xxh_shell, install_repos in xxh_shell_repos.items(): + for rtype, repos in install_repos.items(): + for repo in repos: + repo_dir = pf'{local_xxh_home}/.xxh/{rtype}/{repo}' + repo_local_path = pf'/xxh/{repo}' + + if repo_dir.exists(): + if repo_local_path.exists(): + print(f'Repo {repo}: skip installing from ../{repo}') + else: + print(f'Repo {repo}: skip installing from source') + else: + if repo_local_path.exists(): + print(f'Repo {repo}: replaced from /xxh/{repo}. Do not forget to pull from master!') + repo = f'{repo}+path+{repo_local_path}' + @(xxh) +I @(repo) ++local-xxh-home @(local_xxh_home) + + ssh_opts = ["-o", "StrictHostKeyChecking=accept-new", "-o", "LogLevel=QUIET"] + + if 'xxh-shell-xonsh' in xxh_shell_repos: + print('[Local test xonsh xxh plugins]') + print(f'Remove /root/.xxh_test') + rm -rf /root/.xxh_test + check( + 'Local test xonsh xxh plugins', + $(echo @(xxh) local +iff +hh /root/.xxh_test +lh /root/.xxh_test +I xxh-plugin-xonsh-theme-bar +I xxh-plugin-xonsh-pipe-liner +I xxh-plugin-xonsh-autojump +hf /xxh/xde/tests/xonsh/test_plugins.xsh ), + "1234\n5678\n\n{bar}\n{WHITE}{prompt_end}{NO_COLOR}" + ) + + for shell in xxh_shell_repos.keys(): + for host, h in hosts.items(): + if opt.hosts and host not in opt.hosts: + continue + + print(f'\n[{shell} + {host}]') + + if host == 'local': + server = 'local' + host_home = h['home'] + else: + user = h['user'] + server = user + '@' + host + host_home = h['home'] + + check( + f'Remove {server}:~/.xxh', + $(echo @(h['sshpass']) ssh @(h['ssh_auth']) @(ssh_opts) @(server) "rm -rf ~/.xxh"), + '' + ) + + check( + 'Test connect using ssh', + $(echo @(h['sshpass']) ssh @(h['ssh_auth']) @(ssh_opts) @(server) "echo Test!"), + 'Test!' + ) + + if shell == 'xxh-shell-xonsh': + check( + 'Test install xxh', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +iff +s @(shell) +hf /xxh/xde/tests/xonsh/test_env.py ), + "{{'XXH_HOME': '{host_home}/.xxh/.xxh/shells/xxh-shell-xonsh/build/../../../..', 'PYTHONPATH': '{host_home}/.xxh/.xxh/shells/xxh-shell-xonsh/build/../../../../.local/lib/python3.8/site-packages'}}".format(host_home=host_home) + ) + + # check( + # 'Test AppImage extraction on the host', + # $(echo @(h['sshpass']) ssh @(h['ssh_auth']) @(ssh_opts) @(server) @(xxh_args) @(f"[ -d {host_home}/.xxh/.xxh/shells/xxh-shell-xonsh/build/xonsh-squashfs ] && echo 'extracted' ||echo 'not_extracted'") ), + # 'not_extracted' if 'f' in host.split('_')[-1] else 'extracted' + # ) + + check( + 'Test python', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +hf /xxh/xde/tests/xonsh/test_python.xsh ), + "Python 3.8" + ) + + check( + 'Test xonsh run xonsh script', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +hf /xxh/xde/tests/xonsh/test_xonsh_run_xonsh.xsh +e TESTENV="test env" ), + "123\n.xxh\ntest env" + ) + + check( + 'Test pip upgrade', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +hf /xxh/xde/tests/xonsh/test_pip_upgrade.xsh ), + "" + ) + check( + 'Test pip package install', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +hf /xxh/xde/tests/xonsh/test_pip_package_install.xsh ), + "" + ) + check( + 'Test pip package import', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +hf /xxh/xde/tests/xonsh/test_pip_package_import.xsh ), + "[[1], [2], [3]]" + ) + + # Xontribs + + check( + 'Test xontrib', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +iff +hf /xxh/xde/tests/xonsh/test_xontrib.xsh ), + "autojump installed loaded\nschedule installed loaded" + ) + + check( + 'Test xxh plugins', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +iff +hf /xxh/xde/tests/xonsh/test_plugins.xsh ), + "1234\n5678\n\n{bar}\n{WHITE}{prompt_end}{NO_COLOR}" + ) + + elif shell == 'xxh-shell-zsh': + shell_arg = ['+s', shell] + check( + 'Test zsh env', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +if +hf /xxh/xde/tests/zsh/test_env.zsh +e TESTENV="test env" @(shell_arg) ), + "test zsh .xxh and env=test env" + ) + + check( + 'Test zsh command', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +if +hc @('"echo \'test \\"zsh\\" command\'"') @(xxh_args) ), + 'test "zsh" command' + ) + elif shell == 'xxh-shell-bash': + shell_arg = ['+s', shell] + check( + 'Test bash env', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +if +hf /xxh/xde/tests/bash/test_env.sh +e TESTENV="test env" @(shell_arg) ), + "test bash xxh .xxh and env=test env" + ) + + check( + 'Test bash command', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +if +hc @('"echo \'test \\"bash\\" command\'"') @(shell_arg) ), + 'test "bash" command' + ) + + # $OSH_THEME="morris" + # check( + # 'Test bash seamless', + # $(echo source @(xxh)_xxh/xxh.bash @(h['xxh_auth']) @(server) @(xxh_args) +if +hf /xxh/xde/tests/bash/test_seamless.sh @(shell_arg) ), + # "morris" + # ) + + elif shell == 'xxh-shell-fish': + shell_arg = ['+s', shell] + check( + 'Test fish command', + $(echo @(xxh) @(h['xxh_auth']) @(server) @(xxh_args) +if +hc @('"echo test"') @(shell_arg) ), + "test" + ) + + print('\nDONE') diff --git a/xde/tests/xonsh/test_env.py b/xde/tests/xonsh/test_env.py new file mode 100644 index 0000000..d003c75 --- /dev/null +++ b/xde/tests/xonsh/test_env.py @@ -0,0 +1,18 @@ +import sys, os + +global_settings = {} + +if __name__ == "__main__": + + for e in ['XXH_HOME', 'PIP_TARGET', 'PYTHONPATH']: + if e in os.environ: + global_settings[e] = os.environ[e] + + if len(sys.argv) > 1: + setting_name = sys.argv[1] + if setting_name in global_settings: + print(global_settings[setting_name]) + else: + sys.exit(1) + else: + print(global_settings) diff --git a/xde/tests/xonsh/test_pip_package_import.xsh b/xde/tests/xonsh/test_pip_package_import.xsh new file mode 100644 index 0000000..02ee10d --- /dev/null +++ b/xde/tests/xonsh/test_pip_package_import.xsh @@ -0,0 +1,4 @@ +#!/usr/bin/env xonsh + +import pandas as pd +print(pd.DataFrame([1,2,3]).values.tolist()) \ No newline at end of file diff --git a/xde/tests/xonsh/test_pip_package_install.xsh b/xde/tests/xonsh/test_pip_package_install.xsh new file mode 100644 index 0000000..14f0720 --- /dev/null +++ b/xde/tests/xonsh/test_pip_package_install.xsh @@ -0,0 +1,3 @@ +#!/usr/bin/env xonsh + +pip install @('--verbose' if $XXH_VERBOSE == 2 else '--quiet') --upgrade pandas diff --git a/xde/tests/xonsh/test_pip_upgrade.xsh b/xde/tests/xonsh/test_pip_upgrade.xsh new file mode 100644 index 0000000..fbde32a --- /dev/null +++ b/xde/tests/xonsh/test_pip_upgrade.xsh @@ -0,0 +1,3 @@ +#!/usr/bin/env xonsh + +python -m pip install --quiet --upgrade pip diff --git a/xde/tests/xonsh/test_plugins.xsh b/xde/tests/xonsh/test_plugins.xsh new file mode 100644 index 0000000..dc10f53 --- /dev/null +++ b/xde/tests/xonsh/test_plugins.xsh @@ -0,0 +1,11 @@ +#!/usr/bin/env xonsh + +# xxh-plugin-xonsh-autojump +cd / 2>&1 +cd /tmp 2>&1 + +# xxh-plugin-xonsh-pipe-liner +echo "4321\n8765" | pl "line[::-1]" + +# xxh-plugin-xonsh-theme-bar +echo $PROMPT \ No newline at end of file diff --git a/xde/tests/xonsh/test_python.xsh b/xde/tests/xonsh/test_python.xsh new file mode 100644 index 0000000..19eadc2 --- /dev/null +++ b/xde/tests/xonsh/test_python.xsh @@ -0,0 +1,3 @@ +#!/usr/bin/env xonsh + +echo @($(python --version)[:10]) diff --git a/xde/tests/xonsh/test_xonsh_run_xonsh.xsh b/xde/tests/xonsh/test_xonsh_run_xonsh.xsh new file mode 100644 index 0000000..f7a4aa9 --- /dev/null +++ b/xde/tests/xonsh/test_xonsh_run_xonsh.xsh @@ -0,0 +1 @@ +xonsh /xxh/xde/tests/xonsh/xonsh_script.xsh \ No newline at end of file diff --git a/xde/tests/xonsh/test_xontrib.xsh b/xde/tests/xonsh/test_xontrib.xsh new file mode 100644 index 0000000..f98d6ab --- /dev/null +++ b/xde/tests/xonsh/test_xontrib.xsh @@ -0,0 +1,9 @@ +#!/usr/bin/env xonsh + +xpip install xontrib-autojump > /dev/null 2> /dev/null +xpip install xontrib-schedule > /dev/null 2> /dev/null + +xontrib load autojump > /dev/null 2> /dev/null +xontrib load schedule > /dev/null 2> /dev/null + +xontrib list autojump schedule 2>&1 diff --git a/xde/tests/xonsh/xonsh_script.xsh b/xde/tests/xonsh/xonsh_script.xsh new file mode 100644 index 0000000..8f92650 --- /dev/null +++ b/xde/tests/xonsh/xonsh_script.xsh @@ -0,0 +1,5 @@ +#!/usr/bin/env xonsh + +print('123') +print(pf'{$XXH_HOME}'.resolve().name) +print($TESTENV) \ No newline at end of file diff --git a/xde/tests/zsh/test_env.zsh b/xde/tests/zsh/test_env.zsh new file mode 100644 index 0000000..58c3d02 --- /dev/null +++ b/xde/tests/zsh/test_env.zsh @@ -0,0 +1 @@ +echo "test $ZSH_NAME $(basename -- "$(readlink -f $XXH_HOME)") and env=$TESTENV" diff --git a/xde/xde b/xde/xde new file mode 100755 index 0000000..6976f34 --- /dev/null +++ b/xde/xde @@ -0,0 +1,175 @@ +#!/usr/bin/env xonsh +# PYTHON_ARGCOMPLETE_OK +import os, sys, argparse, yaml, requests, argcomplete +from argcomplete.completers import ChoicesCompleter + +class XxhDevEnv(object): + def __init__(self): + with open('docker-compose.yml') as docker_compose: + self.containers = sorted(yaml.load(docker_compose, Loader=yaml.FullLoader)['services'].keys()) + parser = argparse.ArgumentParser( + usage='xde \n\n' + + 'xxh development environment commands:\n\n' + + ' clone Git clone repos from https://github.com/xxh\n' + + ' build Build the docker containers and git clone the xxh repos if needed\n' + + ' up Docker-compose up the containers\n' + + ' test t Run tests\n' + + ' goto g Open bash by the container name part\n' + + ' start Docker-compose start the containers\n' + + ' stop Docker-compose stop the containers\n' + + ' remove Docker-compose remove the containers\n\n' + + 'Try `./xde --help` to get more info.\n') + parser.add_argument('command', help='Command to run').completer=ChoicesCompleter(('clone', 'build', 'up', 'test', 'goto', 'start', 'stop', 'remove')) + argcomplete.autocomplete(parser) + args = parser.parse_args(sys.argv[1:2]) + if not hasattr(self, args.command): + print('Unrecognized command\n') + parser.print_help() + sys.exit(1) + + getattr(self, args.command)() + + def clone(self): + parser = argparse.ArgumentParser(description='') + parser.add_argument('-n','--name', help=f"Clone repo by name. Example: xxh") + parser.add_argument('-p','--prefix', help=f"Clone repo by prefix. Example: xxh-shell") + parser.add_argument('-s','--substr', help=f"Clone repo by substring. Example: fish") + parser.usage = parser.format_usage().replace('usage: xde ', 'xde clone ') + argcomplete.autocomplete(parser) + args = parser.parse_args(sys.argv[2:]) + + data = requests.get('https://api.github.com/orgs/xxh/repos') + for r in data.json(): + repo_name = r['name'] + + if args.name and not repo_name == args.name: + continue + if args.prefix and not repo_name.startswith(args.prefix): + continue + if args.substr and not (args.substr in repo_name): + continue + + repo_path = p'..' / repo_name + if not repo_path.exists(): + print(f"Git clone {repo_name} to {repo_path}") + git clone -q @(r['clone_url']) @(repo_path) + else: + print(f"Already exists: {repo_path}") + + def build(self): + base_path = pf"{os.path.realpath(__file__)}" + xxh_path = base_path.parent.parent + +# clone_repos = [] +# repo_base_path = 'https://github.com/xxh' + + parser = argparse.ArgumentParser(description='') + parser.add_argument('-v', '--verbose', action='store_true', help=f"Verbose mode") +# parser.add_argument('-rb', '--repo-base', default=repo_base_path, help=f"Repos base path. Default: {repo_base_path}") + parser.usage = parser.format_usage().replace('usage: xde ', 'xde remove ') + argcomplete.autocomplete(parser) + args = parser.parse_args(sys.argv[2:]) + +# repo_base_path = args.repo_base + quiet = [] if args.verbose else ['-q'] + +# for repo in clone_repos: +# if not (xxh_path/repo).exists(): +# print(f'git clone {repo}') +# git clone @(quiet) @(repo_base_path)/@(repo) @(xxh_path/repo) + + for dockerfile in sorted(p'.'.glob('xxh-dev-*.Dockerfile')): + container_name = str(dockerfile).split('.Dockerfile')[0] + print(f'docker build {container_name}') + docker build . -f @(dockerfile) -t xxh/@(container_name) @(quiet) + + def up(self): + parser = argparse.ArgumentParser(description='') + parser.usage = parser.format_usage().replace('xde ', 'xde up ') + argcomplete.autocomplete(parser) + args = parser.parse_args(sys.argv[2:]) + + docker-compose up -d + + def start(self): + parser = argparse.ArgumentParser(description='') + parser.usage = parser.format_usage().replace('xde ', 'xde start ') + argcomplete.autocomplete(parser) + args = parser.parse_args(sys.argv[2:]) + + docker-compose start + + def stop(self): + parser = argparse.ArgumentParser(description='') + parser.usage = parser.format_usage().replace('xde ', 'xde stop ') + argcomplete.autocomplete(parser) + args = parser.parse_args(sys.argv[2:]) + + docker-compose stop + + def remove(self): + parser = argparse.ArgumentParser(description='') + parser.add_argument('-f', '--full', action='store_true', help=f"Full remove: containers, images") + parser.usage = parser.format_usage().replace('usage: xde ', 'xde remove ') + argcomplete.autocomplete(parser) + args = parser.parse_args(sys.argv[2:]) + + docker-compose rm -f -s + if args.full: + containers = ['xxh/'+str(c).replace('.Dockerfile','') for c in p'.'.glob('xxh-dev-*.Dockerfile')] + docker image rm @(containers) + + def t(self): + return self.test() + + def test(self): + it = [] if '--not-interactive' in sys.argv or '-ni' in sys.argv else ['-it'] + r = ![docker exec @(it) xde_start_1 xonsh /xxh/xde/tests/tests.xsh @(sys.argv[2:])] + sys.exit(r.rtn) + + def g(self): + return self.goto() + + def goto(self): + parser = argparse.ArgumentParser(description='') + parser.add_argument('container', help=f"Container name: {', '.join(self.containers)}") + parser.usage = parser.format_usage().replace('usage: xde ', 'xde goto ') + argcomplete.autocomplete(parser) + args = parser.parse_args(sys.argv[2:]) + + match_containers = [c for c in self.containers if args.container in c] + matched_cnt = len(match_containers) + + if matched_cnt == 0: + print(f"Container {args.container} not found among: {', '.join(self.containers)}") + sys.exit(1) + elif matched_cnt == 1 or match_containers[0] == args.container: + args.container = match_containers[0] + else: + print(f"Container '{args.container}' found many times: {', '.join(match_containers)}") + sys.exit(1) + + container_name = f'xde_{args.container}_1' + docker exec -it @(container_name) bash + + def release(self): + parser = argparse.ArgumentParser(description='') + parser.add_argument('-r', '--remove', action='store_true', help=f"Remove build dirs after release") + parser.usage = parser.format_usage().replace('xde ', 'xde release ') + argcomplete.autocomplete(parser) + args = parser.parse_args(sys.argv[2:]) + + cd ../xxh && python3 setup.py sdist bdist_wheel && twine upload dist/* + + if args.remove: + print('Remove build dirs') + rm -rf ./build ./dist ./xxh_xxh.egg-info + + def portable(self): + if p'../xxh-portable'.exists(): + cd ../xxh-portable + docker image rm xxh/xxh-portable-musl-alpine + ./build-xxh-portable-musl-alpine.sh + +if __name__ == '__main__': + XxhDevEnv() diff --git a/xde/xde-test-fast b/xde/xde-test-fast new file mode 100755 index 0000000..1963507 --- /dev/null +++ b/xde/xde-test-fast @@ -0,0 +1,11 @@ +#!/usr/bin/env xonsh + +import os, sys, argparse + +argp = argparse.ArgumentParser(description=f"Fast testing without rebuilding.") +argp.add_argument('-sr', '--skip-repos-update', default=True, action='store_true', help="Skip repos update before test running.") +argp.add_argument('-s', '--shell', default='xxh-shell-xonsh', help="Shells to test") +argp.add_argument('-H', '--hosts', default='ubuntu_k', help="Comma separated hosts list") +opt = argp.parse_args() + +./xde test @(['-sr'] if opt.skip_repos_update else []) --hosts @(opt.hosts) -s @(opt.shell) diff --git a/xde/xxh-dev-centos-k.Dockerfile b/xde/xxh-dev-centos-k.Dockerfile new file mode 100644 index 0000000..8a7b17a --- /dev/null +++ b/xde/xxh-dev-centos-k.Dockerfile @@ -0,0 +1,6 @@ +FROM robertdebock/docker-centos-openssh +# https://github.com/robertdebock/docker-centos-openssh + +RUN yum -y install glibc-locale-source glibc-langpack-en + +ENTRYPOINT ["/bin/sh","-c", "mkdir -p /root/.ssh && cp /xxh/xde/keys/id_rsa.pub /root/.ssh/authorized_keys && chown root:root /root/.ssh/authorized_keys && chmod 0600 /root/.ssh/authorized_keys && /start.sh"] diff --git a/xde/xxh-dev-start-entrypoint.sh b/xde/xxh-dev-start-entrypoint.sh new file mode 100755 index 0000000..6b393a5 --- /dev/null +++ b/xde/xxh-dev-start-entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +for user_dir in /home/*; do + username=`basename $user_dir` + echo Prepare $user_dir + + cd $user_dir + + cp /xxh/xde/keys/id_rsa . + chown $username:$username id_rsa + chmod 0600 id_rsa +done + +tail -f /dev/null diff --git a/xde/xxh-dev-start.Dockerfile b/xde/xxh-dev-start.Dockerfile new file mode 100644 index 0000000..1cdfb6d --- /dev/null +++ b/xde/xxh-dev-start.Dockerfile @@ -0,0 +1,24 @@ +FROM ubuntu:18.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt update && apt install -y openssh-client sshpass rsync wget curl git python3-pip vim mc zsh fish sudo locales +RUN python3 -m pip install --upgrade pip +RUN pip install xonsh==0.9.14 pexpect pyyaml asciinema + +RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + update-locale LANG=en_US.UTF-8 +ENV LANG en_US.UTF-8 + +RUN useradd -m -s $(which bash) ubash +RUN useradd -m -s $(which xonsh) uxonsh +RUN useradd -m -s $(which zsh) uzsh +RUN useradd -m -s $(which fish) ufish + +ADD xxh-dev-start*.sh / +RUN /xxh-dev-start.sh + +ENTRYPOINT ["/xxh-dev-start-entrypoint.sh"] + + diff --git a/xde/xxh-dev-start.sh b/xde/xxh-dev-start.sh new file mode 100755 index 0000000..18ed665 --- /dev/null +++ b/xde/xxh-dev-start.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +ln -s /root /home/root + +cat >/home/root/.bash_history <> .bashrc + elif [[ $username == *"bash"* ]]; then + # https://github.com/ohmybash/oh-my-bash#manual-installation + git clone --depth 1 https://github.com/ohmybash/oh-my-bash .oh-my-bash + cp .bashrc .bashrc.orig + cp .oh-my-bash/templates/bashrc.osh-template .bashrc + sed -i -e 's/font/powerline/g' .bashrc + echo 'export PATH=/xxh/xxh:$PATH' >> .bashrc + + elif [[ $username == *"xonsh"* ]]; then + echo '$PATH=["/xxh/xxh"]+$PATH' >> .xonshrc + elif [[ $username == *"zsh"* ]]; then + # https://github.com/ohmyzsh/ohmyzsh/#manual-installation + git clone --depth 1 https://github.com/ohmyzsh/ohmyzsh.git .oh-my-zsh + cp .oh-my-zsh/templates/zshrc.zsh-template .zshrc + sed -i -e 's/robbyrussell/agnoster/g' .zshrc + sed -i -e 's/(git/(git docker ubuntu/g' .zshrc + + echo 'export PATH=/xxh/xxh:$PATH' >> .zshrc + + elif [[ $username == *"fish"* ]]; then + mkdir -p .config/fish/ + echo 'set PATH /xxh/xxh $PATH' >> .config/fish/config.fish + fi + + chown -R $username:$username $user_dir + +done diff --git a/xde/xxh-dev-ubuntu-k.Dockerfile b/xde/xxh-dev-ubuntu-k.Dockerfile new file mode 100644 index 0000000..7392d73 --- /dev/null +++ b/xde/xxh-dev-ubuntu-k.Dockerfile @@ -0,0 +1,15 @@ +FROM rastasheep/ubuntu-sshd:18.04 +# https://github.com/rastasheep/ubuntu-sshd + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt update && apt install -y vim mc locales +RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + dpkg-reconfigure --frontend=noninteractive locales && \ + update-locale LANG=en_US.UTF-8 +ENV LANG en_US.UTF-8 + +ENTRYPOINT ["/bin/sh","-c", "cp /xxh/xde/keys/id_rsa.pub /root/.ssh/authorized_keys && chown root:root /root/.ssh/authorized_keys && chmod 0600 /root/.ssh/authorized_keys && /usr/sbin/sshd -D"] + + + + diff --git a/xde/xxh-dev-ubuntu-kf.Dockerfile b/xde/xxh-dev-ubuntu-kf.Dockerfile new file mode 100644 index 0000000..642f3ca --- /dev/null +++ b/xde/xxh-dev-ubuntu-kf.Dockerfile @@ -0,0 +1,6 @@ +FROM xxh/xxh-dev-ubuntu-k +# https://github.com/rastasheep/ubuntu-sshd + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt update && apt install -y fuse rsync + diff --git a/requirements.txt b/xxh/requirements.txt similarity index 100% rename from requirements.txt rename to xxh/requirements.txt diff --git a/xxh/xxh b/xxh/xxh index 916537a..d2c751c 100755 --- a/xxh/xxh +++ b/xxh/xxh @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -import os, sys from shutil import which from xxh_xxh.xxh import xxh, eeprint From 5a0b7ea9446cceec83c3f1111501ddf4bc5b9e16 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 15 Mar 2021 08:15:42 +0300 Subject: [PATCH 6/6] wip --- xde/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xde/README.md b/xde/README.md index 3e7fe03..95f304f 100644 --- a/xde/README.md +++ b/xde/README.md @@ -1,6 +1,8 @@ +# Xxh Development Environment (xde) + Development and test environment for [xxh/xxh](https://github.com/xxh/xxh) contains network of docker containers which allow to test the ssh connections and xxh functionality -with or without [AppImage FUSE](https://github.com/AppImage/AppImageKit/wiki/FUSE). +with or without [FUSE](https://github.com/AppImage/AppImageKit/wiki/FUSE). Docker containers in the network: