diff --git a/.gitignore b/.gitignore index 957c88a0d5..4bb769e9f0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,13 @@ *.pyc *~ -/build/ +/build*/ /doc/_build/* /static/SageMenu.mnu /static/all.min.css /static/embedded_sagecell.js +/static/embedded_sagecell.js.map +/static/embedded_sagecell.js.LICENSE.txt /static/images/ /static/jquery.min.js /static/jquery-ui.min.css @@ -16,3 +18,4 @@ /tests/multimechanize/results/ /config.py /sqlite.db +node_modules diff --git a/.gitmodules b/.gitmodules index 10bd4c7b13..e69de29bb2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "submodules/jquery-ui-touch-punch"] - path = submodules/jquery-ui-touch-punch - url = git://github.com/furf/jquery-ui-touch-punch.git diff --git a/Makefile b/Makefile index 0be490ebd9..52fc498120 100644 --- a/Makefile +++ b/Makefile @@ -1,48 +1,35 @@ sage-root := $(shell [ -n "$$SAGE_ROOT" ] && echo "$$SAGE_ROOT" || sage --root || echo "\$$SAGE_ROOT") all-min-js = static/embedded_sagecell.js +all-min-js-map = static/embedded_sagecell.js.map +all-min-js-license = static/embedded_sagecell.js.LICENSE.txt sagecell-css = static/sagecell.css -all-min-css = build/all.min.css embed-css = static/sagecell_embed.css tos-default = templates/tos_default.html tos = templates/tos.html tos-static = static/tos.html -all: submodules $(all-min-js) $(all-min-css) $(embed-css) $(tos-static) +all: $(all-min-js) $(embed-css) $(tos-static) -.PHONY: submodules $(tos-static) - -submodules: - if git submodule status | grep -q ^[+-]; then git submodule update --init > /dev/null; fi +.PHONY: $(tos-static) build: + npm install -rm -r build - cp -a $(sage-root)/local/lib/python2.7/site-packages/notebook/static build - cp $(sage-root)/local/lib/python2.7/site-packages/sagenb/data/sage/js/canvas3d_lib.js \ - static/colorpicker/js/colorpicker.js \ - build - ln -sfn $(sage-root)/local/share/jsmol static/jsmol - ln -sfn $(sage-root)/local/share/threejs static/threejs + npm run build:deps + ln -sfn $(SAGE_VENV)/share/jupyter/nbextensions/jupyter-jsmol/jsmol static/jsmol + ln -sfn $(sage-root)/local/share/threejs-sage/r122 static/threejs ln -sf $(sage-root)/local/share/jmol/appletweb/SageMenu.mnu static/SageMenu.mnu - cp static/jsmol/JSmol.min.nojq.js build/JSmol.js - wget -P build \ - https://raw.githubusercontent.com/sockjs/sockjs-client/master/dist/sockjs.js \ - https://raw.githubusercontent.com/requirejs/domReady/latest/domReady.js \ - https://raw.githubusercontent.com/requirejs/text/latest/text.js - python -c "from matplotlib.backends.backend_webagg_core import FigureManagerWebAgg; print(FigureManagerWebAgg.get_javascript().encode('utf8'))" > build/mpl.js - -$(all-min-js): build $(all-min-css) js/* + cp static/jsmol/JSmol.min.nojq.js build/vendor/JSmol.js + +$(all-min-js): build js/* + npm run build + cp build/embedded_sagecell.js $(all-min-js) + cp build/embedded_sagecell.js.map $(all-min-js-map) + cp build/embedded_sagecell.js.LICENSE.txt $(all-min-js-license) # Host standalone jquery for compatibility with old instructions - cp build/components/jquery/jquery.min.js static - cp submodules/jquery-ui-touch-punch/jquery.ui.touch-punch.min.js build/jquery-ui-tp.js - cp -a js/* build - cd build && r.js -o build.js - cp build/main_build.js $(all-min-js) - -$(all-min-css): build $(sagecell-css) - cp -a build/components/jquery-ui/themes/smoothness/* static - r.js -o cssIn=static/main.css out=$(all-min-css) + cp build/vendor/jquery*.min.js static/jquery.min.js $(embed-css): $(sagecell-css) sed -e 's/;/ !important;/g' < $(sagecell-css) > $(embed-css) diff --git a/README.md b/README.md index f0f69f5717..a0c0878b6f 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ In particular, system packages installed in the base container are listed [here] sudo apt-get install npm # On Debian based systems we need to make an alias sudo ln -s /usr/bin/nodejs /usr/bin/node - sudo npm install -g requirejs ``` 2. Get and build Sage (`export MAKE="make -j8"` or something similar can speed things up): @@ -29,6 +28,10 @@ In particular, system packages installed in the base container are listed [here] ```bash git clone https://github.com/sagemath/sage.git pushd sage + ./bootstrap + ./configure --enable-download-from-upstream-url + # read messages at the end, follow instructions given there. + # possibly install more system packages (using apt-get, if on Debian/Ubuntu) make popd ``` @@ -47,11 +50,14 @@ In particular, system packages installed in the base container are listed [here] ```bash git clone https://github.com/sagemath/sagecell.git pushd sagecell - git submodule update --init --recursive ../sage/sage -sh -c make ``` -Major JavaScript dependencies, including Require.js and CodeMirror.js, are [copied](https://github.com/sagemath/sagecell/blob/master/Makefile#L23) from the [Jupyter notebook](https://github.com/jupyter/notebook) bundled with SageMath. +To build just the Javascript components, from the `sagecell` directory run + +```bash +make static/embedded_sagecell.js +``` # Configuration @@ -71,6 +77,33 @@ Major JavaScript dependencies, including Require.js and CodeMirror.js, are [copi When you want to shut down the server, press `Ctrl-C` in the same terminal. +# Javascript Development + +Javascript source files are compiled using [Webpack](https://webpack.js.org/). Sagecell depends on source files copied +from the Jupyter notebook project. To start development navigate to the `sagecell` source directory and run + +```bash +npm install +npm run build:deps +``` + +After this, all dependencies will be located in the `build/vendor` directory. You can now run + +```bash +npm run build +``` + +to build `build/embedded_sagecell.js` + +or + +```bash +npm run watch +``` + +to build `build/embedded_sagecell.js` and watch files for changes. If a file is changed, `embedded_sagecell.js` will be automatically +rebuilt. + # License See the [LICENSE.txt](LICENSE.txt) file for terms and conditions for usage and a diff --git a/backend_cell.py b/backend_cell.py index d145d1a1c4..b7dae83338 100644 --- a/backend_cell.py +++ b/backend_cell.py @@ -77,14 +77,14 @@ def display_immediately(self, plain_text, rich_output): Example plain text output """ if isinstance(rich_output, OutputPlainText): - return {u'text/plain': rich_output.text.get()}, {} + return {u'text/plain': rich_output.text.get_str()}, {} if isinstance(rich_output, OutputAsciiArt): - return {u'text/plain': rich_output.ascii_art.get()}, {} + return {u'text/plain': rich_output.ascii_art.get_str()}, {} if isinstance(rich_output, OutputLatex): - display_html(rich_output.mathjax()) + display_html(rich_output.latex.get_str()) elif isinstance(rich_output, OutputHtml): - display_html(rich_output.html.get()) + display_html(rich_output.html.get_str()) elif isinstance(rich_output, OutputImageGif): display_file(rich_output.gif.filename(), 'text/image-filename') @@ -97,9 +97,6 @@ def display_immediately(self, plain_text, rich_output): elif isinstance(rich_output, OutputImageSvg): display_file(rich_output.svg.filename(), 'text/image-filename') - elif isinstance(rich_output, OutputSceneCanvas3d): - display_file( - rich_output.canvas3d.filename(), 'application/x-canvas3d') elif isinstance(rich_output, OutputSceneJmol): path = tempfile.mkdtemp(suffix=".jmol", dir=".") os.chmod(path, stat.S_IRWXU + stat.S_IXGRP + stat.S_IXOTH) @@ -167,7 +164,6 @@ def supported_output(self): OutputImagePng, OutputImageSvg, - OutputSceneCanvas3d, OutputSceneJmol, OutputSceneThreejs, #OutputSceneWavefront, @@ -190,5 +186,4 @@ def threejs_offline_scripts(self): """ return """ - """ diff --git a/comm.py b/comm.py deleted file mode 100644 index d727ece18e..0000000000 --- a/comm.py +++ /dev/null @@ -1,9 +0,0 @@ -from ipykernel.comm import Comm - -import sys - -class SageCellComm(Comm): - def __init__(self, *args, **kwargs): - sys._sage_.reset_kernel_timeout(float('inf')) - super(SageCellComm, self).__init__(*args, **kwargs) - diff --git a/config_default.py b/config_default.py index 7bde17bec2..e4c99e1986 100644 --- a/config_default.py +++ b/config_default.py @@ -35,9 +35,9 @@ first_beat = 1.0 # Allowed idling between interactions with a kernel -max_timeout = 60 * 90 +max_timeout = 60 * 15 # Even an actively used kernel will be killed after this time -max_lifespan = 60 * 119 +max_lifespan = 60 * 30 # Recommended settings for kernel providers provider_settings = { @@ -50,7 +50,7 @@ # Also, Sage may allocate huge AS, making this limit pointless: # https://groups.google.com/d/topic/sage-devel/1MM7UPcrW18/discussion "preforked_rlimits": { - "RLIMIT_CPU": 120, # CPU time in seconds + "RLIMIT_CPU": 30, # CPU time in seconds }, } diff --git a/contrib/sagecell-client/sagecell-client.py b/contrib/sagecell-client/sagecell-client.py index 673c645a26..bd72502192 100755 --- a/contrib/sagecell-client/sagecell-client.py +++ b/contrib/sagecell-client/sagecell-client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ A small client illustrating how to interact with the Sage Cell Server, version 2 @@ -25,7 +25,7 @@ def __init__(self, url, timeout=10): # RESPONSE: {"id": "ce20fada-f757-45e5-92fa-05e952dd9c87", "ws_url": "ws://localhost:8888/"} # construct the websocket channel url from that self.kernel_url = '{ws_url}kernel/{id}/'.format(**response) - print self.kernel_url + print(self.kernel_url) websocket.setdefaulttimeout(timeout) self._ws = websocket.create_connection( self.kernel_url + 'channels', @@ -100,4 +100,4 @@ def close(self): url = 'https://sagecell.sagemath.org' a = SageCell(url) import pprint - pprint.pprint(a.execute_request('factorial(2012)')) + pprint.pprint(a.execute_request('factorial(2020)')) diff --git a/contrib/sagecell-client/sagecell-service.py b/contrib/sagecell-client/sagecell-service.py index d604c887e0..1022c50c94 100755 --- a/contrib/sagecell-client/sagecell-service.py +++ b/contrib/sagecell-client/sagecell-service.py @@ -1,12 +1,10 @@ #! /usr/bin/env python3 from datetime import datetime -import json import random +import requests import sys import time -import urllib.parse -import urllib.request retries = 3 @@ -17,13 +15,16 @@ def message(s): while retries: retries -= 1 a, b = random.randint(-2**31, 2**31), random.randint(-2**31, 2**31) - code = 'print({} + {})'.format(a, b) - data = urllib.parse.urlencode(dict(code=code, accepted_tos="true")) - data = data.encode('ascii') + # The handling of temporary files in Sage 9.7 does not allow SageMathCell to + # function properly if there are no regular requests producing temporary + # files. To fight it, we'll generate one during health checks. See + # https://groups.google.com/g/sage-devel/c/jpwUb8OCVVc/m/R4r5bnOkBQAJ + code = 'show(plot(sin)); print({} + {})'.format(a, b) try: - with urllib.request.urlopen( - sys.argv[1] + '/service', data, timeout=35) as f: - reply = json.loads(f.read().decode('ascii')) + r = requests.post(sys.argv[1] + '/service', + data={"code": code, "accepted_tos": "true"}, + timeout=5) + reply = r.json() # Every few hours we have a request that comes back as executed, but the # stdout is not in the dictionary. It seems that the compute message # never actually gets sent to the kernel and it appears the problem is diff --git a/contrib/sphinx/sagecellext.py b/contrib/sphinx/sagecellext.py index f129e9e71e..ae1a17b0f2 100644 --- a/contrib/sphinx/sagecellext.py +++ b/contrib/sphinx/sagecellext.py @@ -28,7 +28,7 @@ .. sagecell:: 1+1 - print "hello world" + print("hello world") """ diff --git a/contrib/sphinx2/README.rst b/contrib/sphinx2/README.rst index 5a3ba935f5..7084a789ac 100644 --- a/contrib/sphinx2/README.rst +++ b/contrib/sphinx2/README.rst @@ -19,15 +19,15 @@ Example of usage:: sage: D = [vector([0,0]), vector([1,0])] sage: @interact sage: def f(A = matrix([[1,1],[-1,1]]), D = '[[0,0],[1,0]]', k=(3..17)): - ... print "Det = ", A.det() + ... print("Det = {}".format(A.det())) ... D = matrix(eval(D)).rows() ... def Dn(k): ... ans = [] ... for d in Tuples(D, k): - ... s = sum(A^n*d[n] for n in range(k)) + ... s = sum(A**n * d[n] for n in range(k)) ... ans.append(s) ... return ans - ... G = points([v.list() for v in Dn(k)],size=50) + ... G = points([v.list() for v in Dn(k)], size=50) ... show(G, frame=True, axes=False) diff --git a/contrib/stats/sagecell-stats.ipynb b/contrib/stats/sagecell-stats.ipynb index 6d8ac8803c..5e2bf3d3cf 100644 --- a/contrib/stats/sagecell-stats.ipynb +++ b/contrib/stats/sagecell-stats.ipynb @@ -54,12 +54,12 @@ " try:\n", " lines.append(parseline(s))\n", " except Exception as E:\n", - " #print i, E\n", + " #print(i, E)\n", " errors+=1\n", "gc.enable()\n", - "print \"Errors: \",errors\n", - "print \"Skipped: \",skipped\n", - "print \"Processed: \",i" + "print(\"Errors: \",errors)\n", + "print(\"Skipped: \",skipped)\n", + "print(\"Processed: \",i)" ], "language": "python", "metadata": {}, @@ -158,9 +158,9 @@ "collapsed": false, "input": [ "notsagecell=d[~d.url.str.contains('^(http|https)://[^.]*.sagemath.org')]\n", - "print len(notsagecell)\n", + "print(len(notsagecell))\n", "c=notsagecell.groupby('url').count().sort('url',ascending=False).take([0],axis=1)\n", - "print len(c)\n", + "print(len(c))\n", "c[:500]" ], "language": "python", @@ -182,7 +182,7 @@ "collapsed": false, "input": [ "daily=d['type'].resample('1D',how='count')\n", - "print daily.describe()\n", + "print(daily.describe())\n", "daily.plot(kind='kde')" ], "language": "python", @@ -221,4 +221,4 @@ "metadata": {} } ] -} \ No newline at end of file +} diff --git a/contrib/vm/README.md b/contrib/vm/README.md index 37b44a950a..4a5fb141cd 100644 --- a/contrib/vm/README.md +++ b/contrib/vm/README.md @@ -1,10 +1,14 @@ # Advanced Installation -Here we describe how to setup a "production" instance of SageCell server. +Here we describe how to setup a "production" instance of SageMathCell server. ## Create the "Enveloping Virtual Machine" (EVM). -This is optional, if you are willing to dedicate a physical machine to SageCell, but it is assumed in scripts and instructions. +This is optional, if you are willing to dedicate a physical machine to SageMathCell. In any case **/var/lib/lxc must be a BTRFS system on an SSD.** + +It does not really matter how you create it. It is up to you also what resources you allocate to it. But something like 4 CPU cores, 32 GB RAM, and 200 GB SSD is a good starting point. + +**The automating scripts are NOT TESTED SINCE 2018.** Adjust them to current versions and your needs or use them just as a guidance, if desired. 1. Configure a package proxy, e.g. Apt-Cacher NG, for the machine that will host EVM. 2. Install KVM and configure it for your account, consult your OS documentation as necessary. @@ -26,7 +30,7 @@ This is optional, if you are willing to dedicate a physical machine to SageCell, ./build_host.sh ``` - (If your `virt-install` does not understand `OSVARIANT=ubuntuxenial`, you may try a different version of Ubuntu here, while keeping the same `LOCATION`, `virt-install --os-variant list` may be useful. Using a different base OS is probably possible, but will likely require further changes to build commands.) + (If your `virt-install` does not understand `OSVARIANT=ubuntu18.04`, you may try a different version of Ubuntu here, while keeping the same `LOCATION`, `osinfo-query os` may be useful. Using a different base OS is probably possible, but will likely require further changes to build commands.) 7. You should get `ssh_host.sh` script that allows you to SSH as root to EVM. If it does not work, you probably need to adjust the IP address in it manually. Note that root password is disabled, you must use the SSH key generated during installation. @@ -44,10 +48,13 @@ This is optional, if you are willing to dedicate a physical machine to SageCell, ``` 2. Adjust it if necessary. In particular, note that the default permalink database server is the public one. -3. Run it. If all goes well, the base OS and the master SageCell container will be created. Expect it to take at least an hour. +3. Run it. The first time it adjusts system configuration and asks you to finish your session and start a new one. Then the base OS and the master SageMathCell container will be created. Expect it to take at least an hour. ```bash - ./container_manager.py | tee first_run.log + ./container_manager.py + exit + ./ssh_host.sh + ./container_manager.py ``` 4. To create several compute nodes behind a load balancer, run @@ -82,7 +89,7 @@ This is optional, if you are willing to dedicate a physical machine to SageCell, ProxyPreserveHost On ``` -2. Configure (restricted) access to EVM:8888 for testing newer versions of SageCell. +2. Configure (restricted) access to EVM:8888 for testing newer versions of SageMathCell. 3. Configure (restricted) access to EVM:9999 for HA-Proxy statistics page. 4. If you are going to run multiple EVMs, consider adjusting `/etc/rsyslog.d/sagecell.conf` in them to collect all logs on a single server. @@ -115,17 +122,17 @@ This is optional, if you are willing to dedicate a physical machine to SageCell, cp github/sagecell/contrib/vm/container_manager.py . && patch container_manager.py local.patch ``` -4. Build the new version of the master container and deploy: +4. Build new versions of all containers and deploy: ```bash - ./container_manager.py -m --deploy + ./container_manager.py -b -s -p -m --deploy ``` Note that after the new version is started, the script will wait for a couple hours to make sure that users in the middle of interacting with the old one have finished their work. 5. If you want to first test the new version while keeping the old one in production, run instead ```bash - ./container_manager.py -m -t + ./container_manager.py -b -s -p -m -t ``` and once you are satisfied with it @@ -134,7 +141,7 @@ This is optional, if you are willing to dedicate a physical machine to SageCell, ./container_manager.py --deploy ``` -6. If you know that only some changes to SageMathCell source code were made, you can skip building Sage and its packages from scratch: `./container_manager.py -p -m` +6. If you know that only some changes to SageMathCell source code were made, you can skip building Sage and its packages from scratch: `./container_manager.py -m` 7. For some other options check the built-in help: `./container_manager.py -h` **If these instructions are unclear or do not work, please let us know!** diff --git a/contrib/vm/build_host.sh b/contrib/vm/build_host.sh index 0f6a9db5ea..8b0ef2d1e3 100755 --- a/contrib/vm/build_host.sh +++ b/contrib/vm/build_host.sh @@ -7,8 +7,8 @@ APTPROXY="http://your-proxy-here-or-delete-proxy-from-preseed/" MAC=52:54:00:5a:9e:c1 # Must start with 52:54:00 VM_NAME=schostvm -OSVARIANT=ubuntu16.04 -LOCATION="http://archive.ubuntu.com/ubuntu/dists/xenial/main/installer-amd64/" +OSVARIANT=ubuntu18.04 +LOCATION="http://archive.ubuntu.com/ubuntu/dists/bionic/main/installer-amd64/" sed "s|APTPROXY|$APTPROXY|" preseed.host > preseed.cfg diff --git a/contrib/vm/compute_node/etc/cron.d/sagecell-cleantmp b/contrib/vm/compute_node/etc/cron.d/sagecell-cleantmp index fd2ccc2328..4513074ff1 100644 --- a/contrib/vm/compute_node/etc/cron.d/sagecell-cleantmp +++ b/contrib/vm/compute_node/etc/cron.d/sagecell-cleantmp @@ -1 +1 @@ -*/30 * * * * root /usr/sbin/tmpreaper 2h /tmp/sagecell +*/30 * * * * root /usr/sbin/tmpreaper --mtime-dir 2h /tmp diff --git a/contrib/vm/compute_node/etc/logrotate.d/0-sagecell b/contrib/vm/compute_node/etc/logrotate.d/0-sagecell new file mode 100644 index 0000000000..fdc8e86589 --- /dev/null +++ b/contrib/vm/compute_node/etc/logrotate.d/0-sagecell @@ -0,0 +1,13 @@ +/var/log/syslog +/var/log/sagecell.log +{ + size 1G + rotate 5 + missingok + notifempty + compress + delaycompress + postrotate + /usr/lib/rsyslog/rsyslog-rotate + endscript +} diff --git a/contrib/vm/compute_node/etc/systemd/system/sagecell.service b/contrib/vm/compute_node/etc/systemd/system/sagecell.service index bfcff474e9..310b64f729 100644 --- a/contrib/vm/compute_node/etc/systemd/system/sagecell.service +++ b/contrib/vm/compute_node/etc/systemd/system/sagecell.service @@ -8,6 +8,7 @@ NotifyAccess=all Restart=always SyslogIdentifier=SageMathCell +ExecStartPre=/root/firewall ExecStartPre=\ -/usr/bin/pkill -u {worker} ;\ /bin/rm -rf /tmp/sagecell ;\ diff --git a/contrib/vm/compute_node/root/firewall b/contrib/vm/compute_node/root/firewall index e10fb3a0b3..f469b605ef 100755 --- a/contrib/vm/compute_node/root/firewall +++ b/contrib/vm/compute_node/root/firewall @@ -1,13 +1,102 @@ #!/usr/bin/env bash -iptables --flush INPUT +ipset create allowed hash:net + +ipset add allowed 130.211.113.153 # Permalinks +ipset add allowed 91.189.88.0/24 # Canonical Group Limited, archive.ubuntu.com +ipset add allowed 185.125.188.0/22 +ipset add allowed 128.174.0.0/16 # University of Illinois, Macaulay2 + +ipset add allowed 160.79.104.10 # api.anthropic.com +ipset add allowed 104.18.26.90 # api.deepseek.com.cdn.cloudflare.net +ipset add allowed 104.18.27.90 +ipset add allowed 162.159.140.245 # api.openai.com +ipset add allowed 172.66.0.243 +ipset add allowed 104.18.18.80 # api.x.ai +ipset add allowed 104.18.19.80 + +ipset add allowed 217.160.0.231 # codetables.de +ipset add allowed 162.255.119.162 # data.astropy.org +ipset add allowed 94.136.40.82 # designtheory.org +ipset add allowed 141.38.0.0/16 # Deutscher Wetterdienst +ipset add allowed 162.125.0.0/16 # Dropbox +ipset add allowed 193.144.0.0/14 # ESA - Villafranca Satellite Tracking Station +ipset add allowed 88.131.0.0/16 # European Centre for Disease Prevention and Control +ipset add allowed 151.101.0.0/16 # Fastly, PyPi.org + +ipset add allowed 140.82.112.0/20 # GitHub +ipset add allowed 185.199.108.0/22 + +ipset add allowed 64.233.160.0/19 # Google services +ipset add allowed 74.125.0.0/16 +ipset add allowed 108.177.0.0/17 +ipset add allowed 142.250.0.0/15 +ipset add allowed 172.217.0.0/16 +ipset add allowed 173.194.0.0/16 +ipset add allowed 192.178.0.0/15 +ipset add allowed 209.85.128.0/17 +ipset add allowed 216.58.192.0/19 + +ipset add allowed 89.107.186.22 # graphclasses.org +ipset add allowed 131.142.0.0/16 # Harvard-Smithsonian Center for Astrophysics + +ipset add allowed 108.128.53.182 # elb.koboroute.org +ipset add allowed 52.214.202.32 +ipset add allowed 54.77.106.83 +ipset add allowed 34.207.37.27 # elb.kobotoolbox.org +ipset add allowed 34.232.35.237 + +ipset add allowed 35.155.106.45 # ljcr.dmgordon.org/mathtrek.eu +ipset add allowed 35.166.140.139 + +ipset add allowed 82.165.72.196 # mathtrek.eu + +ipset add allowed 104.18.22.152 # api.mistral.ai +ipset add allowed 104.18.23.152 + +ipset add allowed 129.164.0.0/16 # NASA +ipset add allowed 137.78.0.0/16 + +ipset add allowed 206.188.192.120 # neilsloane.com +ipset add allowed 104.239.138.29 # oeis.org +ipset add allowed 216.105.38.13 # sourceforge.net +ipset add allowed 108.167.142.230 # tssfl.com +ipset add allowed 104.244.40.0/21 # Twitter +ipset add allowed 130.88.0.0/16 # University of Manchester +ipset add allowed 130.79.0.0/16 # Universite de Strasbourg +ipset add allowed 137.208.0.0/16 # Vienna University of Economics and Business, CRAN +ipset add allowed 208.80.152.0/22 # Wikimedia +ipset add allowed 134.147.222.194 # www.findstat.org +ipset add allowed 35.244.233.98 # www.kaggle.com + +ipset add allowed 69.147.64.0/18 # Yahoo +ipset add allowed 188.125.95.0/24 + + +iptables --flush # Loopback traffic -iptables -A INPUT -i lo -j ACCEPT +iptables -A INPUT -i lo -j ACCEPT +iptables -A OUTPUT -o lo -j ACCEPT # Established connections -iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT +iptables -A OUTPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT + +# Outbound DHCP request +iptables -A OUTPUT -p udp --dport 67:68 --sport 67:68 -j ACCEPT +# Outbound DNS lookups +iptables -A OUTPUT -p udp -m udp --dport 53 -j ACCEPT +# Outbound Network Time Protocol (NTP) requests +iptables -A OUTPUT -p udp --dport 123 --sport 123 -j ACCEPT + # Incoming connections to SageCell server for computations iptables -A INPUT -p tcp --dport 8888 -j ACCEPT # Incoming connections to nginx for static content iptables -A INPUT -p tcp --dport 8889 -j ACCEPT -# And nothing else, including SSH! -iptables -P INPUT DROP + +# Outbound HTTP to allowed hosts +iptables -A OUTPUT -p tcp -m multiport --dport http,https -m state --state NEW -m set --match-set allowed dst -j ACCEPT + +# And nothing else +iptables -P INPUT DROP +iptables -P FORWARD DROP +iptables -P OUTPUT DROP diff --git a/contrib/vm/container_manager.py b/contrib/vm/container_manager.py index 741b779626..8870167d8a 100755 --- a/contrib/vm/container_manager.py +++ b/contrib/vm/container_manager.py @@ -7,6 +7,7 @@ import logging.config import os import pwd +import random import shlex import shutil import stat @@ -27,17 +28,17 @@ # Container names lxcn_base = "base" # OS and packages +lxcn_sage = "sage" # Sage without extra packages lxcn_precell = "precell" # Everything but SageCell and system configuration lxcn_sagecell = "sagecell" # Sage and SageCell lxcn_backup = "sagecell-backup" # Saved master for restoration if necessary lxcn_tester = "sctest" # Accessible via special port, for testing lxcn_prefix = "sc-" # Prefix for main compute nodes -lxcn_version_prefix = "sage-" # Prefix for fixed version compute nodes # Timeout in seconds to wait for a container to shutdown, network to start etc. -timeout = 60 +timeout = 120 # Time after which SageCell should be up and running. -start_delay = 66 +start_delay = 126 # How long to wait after starting new containers before destroying old ones. deploy_delay = 2*60*60 # Two hours to allow all interacts finish "naturally". @@ -50,71 +51,259 @@ repositories = [ ("sagemath", "sage", "master"), ("sagemath", "sagecell", "master"), - ("matplotlib", "basemap", "master"), ] # Packages to be installed in the base container -packages = """ -automake -bison -build-essential -dvipng -epstool -gettext -gfortran -git -imagemagick -iptables -libcairo2-dev -libffi-dev -libsystemd-dev -m4 -nginx -npm -php7.2-fpm -rsyslog-relp -ssh -texlive -texlive-latex-extra -transfig -unattended-upgrades -unzip -wget -""".split() -# Due to (other's) bugs, some packages cannot be installed during installation. -# Let's also use it to separate "standard tools" and "extra stuff". -packages_later = """ -ffmpeg -graphviz -libgeos-dev -libhdf5-dev -libnetcdf-dev -libxml2-dev -libxslt1-dev -octave -""".split() +system_packages = [ +# SageMath prerequisites as of Sage 9.7 +'bc', +'binutils', +'bzip2', +'ca-certificates', +'cliquer', +'cmake', +'curl', +'ecl', +'eclib-tools', +'fflas-ffpack', +'flintqs', +'g++', +'gcc', +'gengetopt', +'gfan', +'gfortran', +'glpk-utils', +'gmp-ecm', +'lcalc', +'libatomic-ops-dev', +'libboost-dev', +'libbraiding-dev', +'libbz2-dev', +'libcdd-dev', +'libcdd-tools', +'libcliquer-dev', +'libcurl4-openssl-dev', +'libec-dev', +'libecm-dev', +'libffi-dev', +'libflint-dev', +'libfplll-dev', +'libfreetype6-dev', +'libgc-dev', +'libgd-dev', +'libgf2x-dev', +'libgiac-dev', +'libgivaro-dev', +'libglpk-dev', +'libgmp-dev', +'libgsl-dev', +'libhomfly-dev', +'libiml-dev', +'liblfunction-dev', +'liblinbox-dev', +'liblrcalc-dev', +'liblzma-dev', +'libm4ri-dev', +'libm4rie-dev', +'libmpc-dev', +'libmpfi-dev', +'libmpfr-dev', +'libncurses5-dev', +'libntl-dev', +'libopenblas-dev', +'libpari-dev', +'libpcre3-dev', +'libplanarity-dev', +'libppl-dev', +'libprimesieve-dev', +'libpython3-dev', +'libqhull-dev', +'libreadline-dev', +'librw-dev', +'libsingular4-dev', +'libsqlite3-dev', +'libssl-dev', +'libsuitesparse-dev', +'libsymmetrica2-dev', +'libz-dev', +'libzmq3-dev', +'libzn-poly-dev', +'m4', +'make', +'nauty', +'ninja-build', +'openssl', +'palp', +'pari-doc', +'pari-elldata', +'pari-galdata', +'pari-galpol', +'pari-gp2c', +'pari-seadata', +'patch', +'perl', +'pkg-config', +'planarity', +'ppl-dev', +'python3', +'python3-venv', +'r-base-dev', +'r-cran-lattice', +'singular', +'singular-doc', +'sqlite3', +'sympow', +'tachyon', +'tar', +'tox', +'xcas', +'xz-utils', +# SageMath development +'autoconf', +'automake', +'git', +'gpgconf', +'libtool', +# 'openssh', not available on Ubuntu 22.04 +'openssh-client', +'pkg-config', +# SageMath recommendations +'default-jdk', +'dvipng', +'ffmpeg', +'imagemagick', +'latexmk', +'libavdevice-dev', +'pandoc', +'tex-gyre', +'texlive-fonts-recommended', +'texlive-lang-cyrillic', +'texlive-lang-english', +'texlive-lang-european', +'texlive-lang-french', +'texlive-lang-german', +'texlive-lang-italian', +'texlive-lang-japanese', +'texlive-lang-polish', +'texlive-lang-portuguese', +'texlive-lang-spanish', +'texlive-latex-extra', +'texlive-xetex', +# SageMath optional +'4ti2', +'clang', +'coinor-cbc', +'coinor-libcbc-dev', +'graphviz', +'libfile-slurp-perl', +'libgraphviz-dev', +'libigraph-dev', +'libisl-dev', +'libjson-perl', +'libmongodb-perl', +'libnauty-dev', +'libperl-dev', +'libpolymake-dev', +'libsvg-perl', +'libterm-readkey-perl', +'libterm-readline-gnu-perl', +'libxml-libxslt-perl', +'libxml-writer-perl', +'libxml2-dev', +'lrslib', +'pari-gp2c', +'pdf2svg', +# 'polymake', triggers firefox snap that does not work in containers +'texinfo', +# SageMathCell +'bison', +'build-essential', +'epstool', +'fig2dev', +'gettext', +'gnuplot', +'ipset', +'iptables', +'libcairo2-dev', +'libgeos-dev', +'libhdf5-dev', +'libnetcdf-dev', +'libopenmpi-dev', +'libopenmpi3', +'libproj-dev', +'libsnappy-dev', +'libsystemd-dev', +'libxslt1-dev', +'macaulay2', +'nginx', +'npm', +'octave', +'octave-econometrics', +'octave-statistics', +'octave-symbolic', +'php8.3-fpm', +'proj-bin', +'python3-requests', +'rsyslog-relp', +'ssh', +'texlive', +'tk-dev', +'tmpreaper', +'unattended-upgrades', +'unzip', +'wget', +# R packages +'r-cran-desolve', +'r-cran-ggally', +'r-cran-ggeffects', +'r-cran-ggplot2', +'r-cran-lazyeval', +'r-cran-pracma', +'r-cran-reticulate', +'r-cran-rhandsontable', +'r-cran-rms', +'r-cran-survey', +'r-cran-tidyverse', +] + +# R packages that are not available as system ones +R_packages = [ +"flextable", +"formattable", +"ggformula", +"glmmTMB", +"gt", +"gtExtras", +"huxtable", +"kableExtra", +"mosaic", +"pixiedust", +"reactable", +"reactablefmtr", +"swirl", +] # Optional Sage packages to be installed sage_optional_packages = [ "4ti2", "biopython", +"bliss", "cbc", -"cryptominisat", "database_cremona_ellcurve", "database_jones_numfield", "database_odlyzko_zeta", "database_symbolic_data", "dot2tex", # needs graphviz +"fricas", "gap_packages", "gap3", -"guppy", +"jmol", +"jupyter_jsmol", "latte_int", "lie", # needs bison "lrslib", "mcqd", "normaliz", -"nose", -"ore_algebra", "pari_elldata", "pari_galpol", "pari_nftables", @@ -123,41 +312,136 @@ "pynormaliz", "qepcad", "saclib", -"topcom", +"tides", +#"topcom", Does not work as of November 2022 with relying on system packages ] # Python packages to be installed into Sage (via pip) python_packages = [ # Dependencies of SageMathCell +"comm", "lockfile", "paramiko", +"psutil", "sockjs-tornado", "git+https://github.com/systemd/python-systemd.git", # Optional +"future", # fipy does not work without it installed first +#"--no-build-isolation git+https://github.com/abelfunctions/abelfunctions", downgrades numpy +"admcycles", +"altair", "APMonitor", +"astropy", +"astroquery", +#"autoviz", downgrades numpy +"bioinfokit", "bitarray", +"bokeh", +"calplot", +"cartopy", +"chart_studio", +"colorlog", +"covid-daily", +"cramjam", +"cufflinks", +"dash", +"dask[array]", +"drawdata", +"dropbox", +"duckdb", +"emoji", +"galgebra", +"geopandas", +"geoplot", +"getdist", "ggplot", +"gif", +#"giotto-tda", wants sudo +"google-api-python-client", +"google-generativeai", +"graphviz", +"gspread", +"fipy", +"folium", +"healpy", "h5py", "husl", +"itikz", +"july", +"keras", +"keyring", +"koboextractor", +"langchain", +"langchain-openai", +"langserve", +"langserve[all]", +#"lenstools", complaints there is no numpy +"lhsmdu", +"litellm", "lxml", +"manimlib", +"mapclassify", "mathchem", +"mistralai", +"mpi4py", +"msedge-selenium-tools", "munkres", +"nest_asyncio", "netcdf4", +"nltk", "numexpr", +"oauth2client", "oct2py", +"openai", +"openpyxl", "pandas", +"pandas-profiling", "patsy", +"plotnine", +"plotly", +"polars", +"pretty_html_table", +"pydot", +"pyforest", +"pygnuplot", +"PyPDF4", "pyproj", +"pyswarms", +"python-snappy", +"python-ternary", +"pyvo", +"qiskit", +"qiskit[nature]", "requests", "scikit-image", "scikit-learn", -"scimath", +"scikit-tda", +#"scimath", does not build +"scrapy", "seaborn", +"selenium", "Shapely", "SimPy", +"snappy", +"spacy", +"SpeechRecognition", +"spiceypy", "statsmodels", -"surface_dynamics", +#"surface_dynamics", does not build +"sweetviz", "tables", +"tbcontrol", +#"theano", does not build +"tikzplotlib", +"torch", +"transformers", +"tweepy", +"twint", +"vega_datasets", +"WeasyPrint", +"wordcloud", +"xarray", +"xlrd", "moss", # This one only complains about missing dependencies ] @@ -196,7 +480,7 @@ # HA-Proxy configuration is regenerated every time the script is run. HAProxy_header = """\ -# Default from Ubuntu 18.04 LTS +# Default from Ubuntu 22.04 LTS global log /dev/log local0 log /dev/log local1 notice @@ -211,13 +495,10 @@ ca-base /etc/ssl/certs crt-base /etc/ssl/private - # Default ciphers to use on SSL-enabled listening sockets. - # For more information, see ciphers(1SSL). This list is from: - # https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ - # An alternative list with additional directives can be obtained from - # https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=haproxy - ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS - ssl-default-bind-options no-sslv3 + # See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate + ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 + ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets defaults log global @@ -239,7 +520,7 @@ option http-server-close option redispatch timeout client-fin 50s - timeout tunnel 2h + timeout tunnel 30m """ # {suffix} {port} {hostname} {peer_port} have to be set once @@ -247,7 +528,8 @@ HAProxy_section = r""" frontend http{suffix} bind *:{port} - reqrep ^([^\ \t]*[\ \t])(/embedded_sagecell\.js[\ \t].*) \1/static\2 + rate-limit sessions 10 + http-request replace-path (/embedded_sagecell\.js.*) /static\1 if { url_beg /embedded_sagecell } use_backend static{suffix} if { path_beg /static } use_backend compute{suffix} monitor-uri /?healthcheck @@ -260,7 +542,8 @@ server {node} {ip}:8889 id {id} check backend compute{suffix} - stick-table type string len 36 size 1m expire 2h peers local{suffix} + balance leastconn + stick-table type string len 36 size 1m expire 30m peers local{suffix} stick on urlp(CellSessionID) stick match req.hdr(Jupyter-Kernel-ID) stick store-response res.hdr(Jupyter-Kernel-ID) @@ -310,7 +593,7 @@ def communicate(command, message): msg = "{} failed".format(command) log.error(msg) raise RuntimeError(msg) - + def timer_delay(delay, test=None): r""" @@ -350,7 +633,6 @@ def update_repositories(): git("checkout " + branch) if call("git symbolic-ref -q HEAD") == 0: git("pull") - git("submodule update --init --recursive") os.chdir(os.pardir) os.chdir(os.pardir) @@ -393,27 +675,40 @@ def setup_container_users(): "--disabled-password {worker}") shome = os.path.join("/home", users["server"]) - whome = os.path.join("/home", users["worker"]) + os.chmod(shome, stat.S_IRWXU | + stat.S_IRGRP | stat.S_IXGRP | + stat.S_IROTH | stat.S_IXOTH) os.chdir(shome) os.setegid(users["GID"]) os.seteuid(users["server_ID"]) os.mkdir(".ssh", 0o700) - check_call("ssh-keygen -q -N '' -f .ssh/id_rsa") + check_call("ssh-keygen -t ed25519 -q -N '' -f .ssh/id_ed25519") + whome = os.path.join("/home", users["worker"]) os.chdir(whome) os.setuid(0) os.seteuid(users["worker_ID"]) + os.mkdir(".cache", 0o700) + os.mkdir(".sage") os.mkdir(".ssh", 0o700) - files_to_lock = ".ssh .bashrc .bash_profile .bash_logout .profile" - check_call("touch " + files_to_lock) + files_to_lock = [ + ".cache/pip", + ".sage/local", + ".ssh", + ".bash_logout", + ".bash_profile", + ".bashrc", + ".profile", + ] + check_call(" ".join(["touch"] + files_to_lock)) os.setuid(0) - shutil.copy2(os.path.join(shome, ".ssh/id_rsa.pub"), + shutil.copy2(os.path.join(shome, ".ssh/id_ed25519.pub"), ".ssh/authorized_keys") os.chown(".ssh/authorized_keys", users["worker_ID"], users["GID"]) # Get the localhost in the known_hosts file. check_call("su -l {server} -c " "'ssh -q -oStrictHostKeyChecking=no {worker}@localhost whoami'") - for f in files_to_lock.split(): + for f in files_to_lock: check_call("chattr -R +i " + f) @@ -436,6 +731,8 @@ def install_sage(): shutil.move("github/sage", ".") os.chdir("sage") log.info("compiling Sage") + check_call("./bootstrap") + check_call("./configure") check_call("make") communicate("./sage", r""" # make appropriate octave directory @@ -450,20 +747,18 @@ def install_packages(): Assuming Sage is already installed, install optional packages. """ become_server() + os.chdir("sage") log.info("installing optional Sage packages") for package in sage_optional_packages: - check_call("sage/sage -i -y {}".format(package)) - # And we also install basemap - log.info("installing basemap in Sage") - shutil.move("github/basemap", ".") - os.chdir("basemap") - check_call("../sage/sage setup.py install") - os.chdir("..") - + check_call("./sage -i -y {}".format(package)) log.info("installing pip packages") - check_call("sage/sage -pip install --upgrade pip") + check_call("./sage -pip install --upgrade pip") + numpy_ver = check_output("./sage -c 'import numpy; print(numpy.__version__)'").strip() + log.info(f"numpy version is expected to stay at {numpy_ver}") for package in python_packages: - check_call("sage/sage -pip install {}".format(package)) + # Many packages may downgrade numpy, so we force it to be at the Sage version + check_call(f"./sage -pip install numpy=={numpy_ver} {package}") + os.chdir("..") def install_sagecell(): @@ -505,8 +800,6 @@ def adjust_names(file): name = os.path.join(root, file) adjust_names(shutil.copy(name, name[1:])) check_call("systemctl enable sagecell") - with open("/etc/network/interfaces", "a") as f: - f.write(" up /root/firewall\n") class SCLXC(object): @@ -518,7 +811,7 @@ def __init__(self, name): self.name = name self.c = lxc.Container(self.name) - def clone(self, clone_name, autostart=False, update=False): + def clone(self, clone_name, autostart=False, update=True): r""" Clone self, create a base container and destroy old clone if necessary. """ @@ -535,7 +828,9 @@ def clone(self, clone_name, autostart=False, update=False): if autostart: clone.c.set_config_item("lxc.start.auto", "1") clone.c.set_config_item("lxc.start.delay", str(start_delay)) - clone.c.save_config() + clone.c.set_config_item("lxc.net.0.hwaddr", + "02:00:" + ":".join(["%02x" % random.randint(0, 255) for _ in range(4)])) + clone.c.save_config() logdir = clone.c.get_config_item("lxc.rootfs.path") + "/var/log/" for logfile in ["sagecell.log", "sagecell-console.log"]: if os.path.exists(logdir + logfile): @@ -544,31 +839,106 @@ def clone(self, clone_name, autostart=False, update=False): def create(self): r""" - Create a base contrainer, destroy old one if necessary. + Create this container based on the previous one, destroy old one if necessary. + + It is the logical sequence of creating a fully configured SageMathCell container + from scratch, but broken into several steps. Previous steps for the current container + are performed if necessary, based on names. Random name is assumed to be a copy + of "the end result". """ self.destroy() log.info("creating %s", self.name) - # Try to automatically pick up proxy from host - os.environ["HTTP_PROXY"] = "apt" - if not self.c.create( - "download", 0, - {"dist": "ubuntu", "release": "bionic", "arch": "amd64"}, - "btrfs"): - raise RuntimeError("failed to create " + self.name) - os.environ.unsetenv("HTTP_PROXY") - - log.info("installing packages") - self.inside("apt install -y " + " ".join(packages)) - self.inside("/usr/sbin/deluser ubuntu --remove-home") - log.info("installing later packages") - self.inside("apt install -y " + " ".join(packages_later)) - # Need to preseed or there will be a dialog - self.inside(communicate, "/usr/bin/debconf-set-selections", - "tmpreaper tmpreaper/readsecurity note") - self.inside("apt install -y tmpreaper") - log.info("installing npm packages") - self.inside("npm install -g requirejs") - self.update() + if self.name == lxcn_base: + # From scratch + # Try to automatically pick up proxy from host + os.environ["HTTP_PROXY"] = "apt" + if not self.c.create( + "download", 0, + {"dist": "ubuntu", "release": "noble", "arch": "amd64"}, + "btrfs"): + raise RuntimeError("failed to create " + self.name) + os.environ.pop("HTTP_PROXY") + + self.update() + # Need to preseed or there will be a dialog + self.inside(communicate, "/usr/bin/debconf-set-selections", + "tmpreaper tmpreaper/readsecurity note") + log.info("installing packages") + self.inside("apt install -y " + " ".join(system_packages)) + # Relies on perl, so has to be after package installation + self.inside("/usr/sbin/deluser ubuntu --remove-home") + log.info("installing R packages") + for package in R_packages: + self.inside(f"""Rscript -e 'install.packages("{package}")'""") + self.inside(f"""Rscript -e 'library("{package}")'""") + elif self.name == lxcn_sage: + self.c = SCLXC(lxcn_base).clone(lxcn_sage).c + create_host_users() + self.inside(setup_container_users) + # FIXME: work with temp folders properly + self.inside(os.mkdir, "/tmp/sagecell", 0o730) + self.inside(os.chown, "/tmp/sagecell", + users["server_ID"], users["GID"]) + self.inside(os.chmod, "/tmp/sagecell", stat.S_ISGID) + # Copy repositories into container + update_repositories() + log.info("uploading repositories to %s", self.name) + root = self.c.get_config_item("lxc.rootfs.path") + home = os.path.join(root, "home", users["server"]) + dot_cache = os.path.join(home, ".cache") + shutil.copytree("github", os.path.join(home, "github"), symlinks=True) + self.inside("chown -R {server}:{group} /home/{server}/github") + try: + shutil.copytree("dot_cache", dot_cache, symlinks=True) + self.inside("chown -R {server}:{group} /home/{server}/.cache") + except FileNotFoundError: + pass + self.inside(install_sage) + elif self.name == lxcn_precell: + self.c = SCLXC(lxcn_sage).clone(lxcn_precell).c + self.inside(install_packages) + # Remove old versions of packages + root = self.c.get_config_item("lxc.rootfs.path") + home = os.path.join(root, "home", users["server"]) + dot_cache = os.path.join(home, ".cache") + upstream = os.path.join(home, "sage/upstream") + packages = dict() + for f in os.listdir(upstream): + filename = os.path.join(upstream, f) + name = f.split("-", 1)[0] + if name not in packages: + packages[name] = [] + packages[name].append((os.stat(filename).st_mtime, filename)) + for package in packages.values(): + package.sort() + package.pop() + for _, filename in package: + os.remove(filename) + try: + shutil.rmtree("github/sage/upstream") + except FileNotFoundError: + pass + shutil.move(upstream, "github/sage/upstream") + try: + shutil.rmtree("dot_cache") + except FileNotFoundError: + pass + shutil.copytree(dot_cache, "dot_cache", symlinks=True) + elif self.name == lxcn_sagecell: + self.c = SCLXC(lxcn_precell).clone(lxcn_sagecell).c + self.inside("su -c 'git -C /home/{server}/github/sagecell pull' {server}") + self.inside(install_sagecell) + self.inside(install_config_files) + self.c.set_config_item("lxc.cgroup.memory.limit_in_bytes", "8G") + self.c.save_config() + self.shutdown() + # Let first-time tasks to run and complete. + self.start() + timer_delay(start_delay) + else: + # If the name is not recognized as some intermediate step, we assume + # that a copy of the fully built SageMathCell is desired + self.c = SCLXC(lxcn_sagecell).clone(self.name).c def destroy(self): r""" @@ -609,77 +979,6 @@ def wrapper(): raise RuntimeError("failed to execute {} with arguments {}" .format(command, args)) - def prepare_for_sagecell(self, keeprepos=False): - r""" - Set up everything necessary for SageCell installation. - - INPUT: - - - ``keeprepos`` -- if ``True``, GitHub repositories will NOT be updated - and set to proper state (useful for development). - """ - create_host_users() - self.inside(setup_container_users) - # FIXME: work with temp folders properly - self.inside(os.mkdir, "/tmp/sagecell", 0o730) - self.inside(os.chown, "/tmp/sagecell", - users["server_ID"], users["GID"]) - self.inside(os.chmod, "/tmp/sagecell", stat.S_ISGID) - # Copy repositories into container - if not keeprepos: - update_repositories() - log.info("uploading repositories to %s", self.name) - root = self.c.get_config_item("lxc.rootfs.path") - home = os.path.join(root, "home", users["server"]) - shutil.copytree("github", os.path.join(home, "github"), symlinks=True) - self.inside("chown -R {server}:{group} /home/{server}/github") - dot_cache = os.path.join(home, ".cache") - try: - shutil.copytree("dot_cache", dot_cache, symlinks=True) - self.inside("chown -R {server}:{group} /home/{server}/.cache") - except FileNotFoundError: - pass - self.inside(install_sage) - self.inside(install_packages) - # Remove old versions of packages - upstream = os.path.join(home, "sage/upstream") - packages = dict() - for f in os.listdir(upstream): - filename = os.path.join(upstream, f) - name = f.split("-", 1)[0] - if name not in packages: - packages[name] = [] - packages[name].append((os.stat(filename).st_mtime, filename)) - for package in packages.values(): - package.sort() - package.pop() - for _, filename in package: - os.remove(filename) - try: - shutil.rmtree("github/sage/upstream") - except FileNotFoundError: - pass - shutil.move(upstream, "github/sage/upstream") - try: - shutil.rmtree("dot_cache") - except FileNotFoundError: - pass - shutil.copytree(dot_cache, "dot_cache", symlinks=True) - - def install_sagecell(self): - r""" - Set up SageCell to run on startup. - """ - self.inside(install_sagecell) - self.inside(install_config_files) - self.c.set_config_item("lxc.cgroup.memory.limit_in_bytes", "8G") - self.c.save_config() - self.shutdown() - # Let first-time tasks to run and complete. - self.start() - timer_delay(start_delay) - self.shutdown() - def ip(self): self.start() return self.c.get_ips()[0] @@ -783,7 +1082,7 @@ def restart_haproxy(names, backup_names=[]): check_call("systemctl start haproxy") -logging.config.dictConfig(yaml.load(""" +logging.config.dictConfig(yaml.safe_load(""" version: 1 formatters: file: @@ -811,19 +1110,16 @@ def restart_haproxy(names, backup_names=[]): epilog=""" Missing necessary containers are always created automatically. - Default action without any options is to make sure that the master - container is present and update its system packages. - This script always overwrites system-wide HA-proxy configuration file and restarts HA-Proxy to resolve container names to new IP addresses.""") -parser.add_argument("-b", "--base", action="store_true", - help="rebuild 'OS and standard packages' container") -parser.add_argument("--keeprepos", action="store_true", - help="keep GitHub repositories at their present state") -parser.add_argument("-p", "--useprecell", action="store_true", - help="don't rebuild Sage and extra packages for master") parser.add_argument("--savemaster", action="store_true", help="save existing master container") +parser.add_argument("-b", "--base", action="store_true", + help="rebuild 'OS and standard packages' container") +parser.add_argument("-s", "--sage", action="store_true", + help="rebuild Sage container") +parser.add_argument("-p", "--precell", action="store_true", + help="rebuild container with extra packages") group = parser.add_mutually_exclusive_group() group.add_argument("-m", "--master", action="store_true", help="rebuild 'Sage and SageCell' container") @@ -851,35 +1147,23 @@ def restart_haproxy(names, backup_names=[]): f.write(rsyslog_conf) check_call("systemctl restart rsyslog") -# Main chain: base -- precell -- (sagecell, backup) -base = SCLXC(lxcn_base) -if args.base: - base.create() - -sagecell = SCLXC(lxcn_sagecell) +# Main chain: base -- sage -- precell -- (sagecell, backup) -- sc-NA/sc-NB if args.savemaster: - sagecell.clone(lxcn_backup) + SCLXC(lxcn_backup).create() +if args.base: + SCLXC(lxcn_base).create() +if args.sage: + SCLXC(lxcn_sage).create() +if args.precell: + SCLXC(lxcn_precell).create() +if args.master: + SCLXC(lxcn_sagecell).create() if args.restoremaster: - sagecell = SCLXC(lxcn_backup).clone(lxcn_sagecell) - -if args.master or not sagecell.is_defined(): - precell = SCLXC(lxcn_precell) - if precell.is_defined() and args.useprecell: - precell.update() - if not args.keeprepos: - precell.inside( - "su -c 'git -C /home/{server}/github/sagecell pull' {server}") - else: - precell = base.clone(lxcn_precell, update=True) - precell.prepare_for_sagecell(args.keeprepos) - sagecell = precell.clone(lxcn_sagecell) - sagecell.install_sagecell() -else: - sagecell.update() + SCLXC(lxcn_backup).clone(lxcn_sagecell) # Autostart containers: tester and deployed nodes. if args.tester: - sagecell.clone(lxcn_tester, autostart=True).start() + SCLXC(lxcn_sagecell).clone(lxcn_tester, autostart=True).start() A_names = ["{}{}{}".format(lxcn_prefix, n, "A") for n in range(number_of_compute_nodes)] @@ -893,9 +1177,11 @@ def restart_haproxy(names, backup_names=[]): up_names, old_names = [], A_names if args.deploy: + sagecell = SCLXC(lxcn_sagecell) + sagecell.update() up_names, old_names = old_names, up_names for n in up_names: - sagecell.clone(n, autostart=True).start() + sagecell.clone(n, autostart=True, update=False).start() log.info("waiting for new containers to fully initialize...") timer_delay(start_delay) old_nodes = list(map(SCLXC, old_names)) diff --git a/contrib/vm/preseed.host b/contrib/vm/preseed.host index 6a840d01ee..f354b7b863 100644 --- a/contrib/vm/preseed.host +++ b/contrib/vm/preseed.host @@ -77,7 +77,7 @@ d-i passwd/make-user boolean false tasksel tasksel/first multiselect openssh-server, server, standard # Individual additional packages to install -d-i pkgsel/include string haproxy lxc python3-psutil python3-yaml rsyslog-relp +d-i pkgsel/include string haproxy lxc python3-lxc python3-psutil rsyslog-relp # Whether to upgrade packages after debootstrap. # Allowed values: none, safe-upgrade, full-upgrade diff --git a/db.py b/db.py index 6f6e066914..c659b7308e 100644 --- a/db.py +++ b/db.py @@ -12,7 +12,7 @@ class DB(object): Abstract base class for database adaptors. """ - def add(self, code, language, interacts, callback): + async def add(self, code, language, interacts): """ Add an entry to the database. @@ -24,12 +24,13 @@ def add(self, code, language, interacts, callback): - ``interacts`` -- a string - - ``callback`` -- a function accepting a single string argument, - the identifier key for the entry + OUTPUT: + + - a string -- the identifier key for the entry """ raise NotImplementedError - def get(self, key, callback): + async def get(self, key): """ Retrieve the entry from the database matching ``key``. @@ -37,7 +38,8 @@ def get(self, key, callback): - ``key`` -- a string - - ``callback`` -- a function accepting three string arguments: the code, - the language, and the interact state. + OUTPUT: + + - a tuple of three strings: the code, the language, the interact state. """ raise NotImplementedError diff --git a/db_sqlalchemy.py b/db_sqlalchemy.py index 006215e8a7..e2835d7892 100644 --- a/db_sqlalchemy.py +++ b/db_sqlalchemy.py @@ -8,8 +8,7 @@ import string from sqlalchemy import create_engine, Column, Integer, String, DateTime -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm import declarative_base, sessionmaker from sqlalchemy.exc import IntegrityError import db @@ -65,7 +64,7 @@ def __init__(self, db_file): Base.metadata.create_all(self.engine) self.dbsession = self.SQLSession() - def add(self, code, language, interacts, callback): + async def add(self, code, language, interacts): """ See :meth:`db.DB.add` """ @@ -85,9 +84,9 @@ def add(self, code, language, interacts, callback): self.dbsession.rollback() else: break - callback(ident) + return ident - def get(self, key, callback): + async def get(self, key): """ See :meth:`db.DB.get` """ @@ -96,4 +95,4 @@ def get(self, key, callback): raise LookupError msg.requested = ExecMessage.requested + 1 self.dbsession.commit() - callback(msg.code, msg.language, msg.interacts) + return (msg.code, msg.language, msg.interacts) diff --git a/db_web.py b/db_web.py index b224928e4a..f1545cfe18 100644 --- a/db_web.py +++ b/db_web.py @@ -20,33 +20,26 @@ class DB(db.DB): def __init__(self, url): self.url = url - def add(self, code, language, interacts, callback): + async def add(self, code, language, interacts): """ See :meth:`db.DB.add` """ - body = urllib.urlencode({ + body = urllib.parse.urlencode({ "code": code.encode("utf8"), "language": language.encode("utf8"), "interacts": interacts.encode("utf8")}) - - def cb(response): - if response.code != 200: - raise RuntimeError("Error in response") - callback(json.loads(response.body)["query"]) - http_client = tornado.httpclient.AsyncHTTPClient() - http_client.fetch(self.url, cb, method="POST", body=body, - headers={"Accept": "application/json"}) + response = await http_client.fetch( + self.url, method="POST", body=body, + headers={"Accept": "application/json"}) + return json.loads(response.body)["query"] - def get(self, key, callback): + async def get(self, key): """ See :meth:`db.DB.get` """ - def cb(response): - if response.code != 200: - raise LookupError("Code lookup produced error") - callback(*json.loads(response.body)) - http_client = tornado.httpclient.AsyncHTTPClient() - http_client.fetch("{}?q={}".format(self.url, key), cb, method="GET", - headers={"Accept": "application/json"}) + response = await http_client.fetch( + "{}?q={}".format(self.url, key), method="GET", + headers={"Accept": "application/json"}) + return json.loads(response.body) diff --git a/doc/embedding.rst b/doc/embedding.rst index 7edcc343a2..f3fa2f7e86 100644 --- a/doc/embedding.rst +++ b/doc/embedding.rst @@ -42,7 +42,7 @@ like):