WIP: Split compute controllers to cloud specific files

.github/workflows/ci.yml

@@ -4,12 +4,48 @@ on:
   push:
     branches:
       - master
-  # pull_request:
+  pull_request:
   workflow_dispatch:
 
 jobs:
+  validate-python:
+    name: Validate Python Code
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install mypy pyinstaller
+
+      - name: Run mypy type checking
+        run: |
+          mypy src/mist/api
+
+      - name: Compile Python files
+        run: |
+          find src/mist/api -name "*.py" -exec python -m py_compile {} \;
+
+      - name: Try PyInstaller build
+        run: |
+          cd src/mist/api
+          pyinstaller --onefile --clean --noconfirm --log-level DEBUG \
+            --hidden-import=mist.api.clouds.controllers.compute.base \
+            --hidden-import=mist.api.clouds.controllers.base \
+            --hidden-import=mist.api.concurrency.models \
+            --hidden-import=mist.api.tag.models \
+            clouds/controllers/compute/base.py
+
   build_mist_image:
     name: Build Mist Image
+    needs: validate-python
     runs-on: ubuntu-latest
     env:
       GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "libcloud"]
 	path = lc
 	url = https://github.com/apache/libcloud.git
-[submodule "paramiko"]
-	path = paramiko
-	url = https://github.com/paramiko/paramiko.git
 [submodule "v2"]
 	path = v2
 	url = ../mist-api-v2.git

Dockerfile

@@ -1,16 +1,16 @@
-FROM python:3.11-slim-bullseye
+FROM python:3.13-slim-bullseye
 
 # Install libvirt which requires system dependencies.
 RUN apt update && \
     apt install -y git build-essential g++ gcc cargo gnupg ca-certificates \
-    libssl-dev libffi-dev libvirt-dev libxml2-dev libxslt-dev zlib1g-dev vim \
-    libmemcached-dev procps netcat wget curl jq inetutils-ping && \
+    libssl-dev libffi-dev libvirt-dev libxml2-dev libxslt1-dev zlib1g-dev vim \
+    procps netcat wget curl jq inetutils-ping && \
     rm -rf /var/lib/apt/lists/*
 
-RUN wget https://dl.influxdata.com/influxdb/releases/influxdb-1.8.4-static_linux_amd64.tar.gz && \
-    tar xvfz influxdb-1.8.4-static_linux_amd64.tar.gz && rm influxdb-1.8.4-static_linux_amd64.tar.gz
+RUN wget https://dl.influxdata.com/influxdb/releases/influxdb2-2.7.11_linux_amd64.tar.gz && \
+    tar xvfz influxdb2-2.7.11_linux_amd64.tar.gz && rm influxdb2-2.7.11_linux_amd64.tar.gz
 
-RUN ln -s /influxdb-1.8.4-1/influxd /usr/local/bin/influxd && \
+RUN ln -s /influxdb2-2.7.11/usr/bin/influxd /usr/local/bin/influxd && \
     ln -s /usr/bin/pip3 /usr/bin/pip && \
     ln -s /usr/bin/python3 /usr/bin/python
 
@@ -20,25 +20,22 @@ RUN ln -s /influxdb-1.8.4-1/influxd /usr/local/bin/influxd && \
 
 RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir --upgrade setuptools && \
-    pip install libvirt-python==8.8.0 uwsgi==2.0.21 && \
+    pip install libvirt-python uwsgi && \
     pip install --no-cache-dir ipython ipdb flake8 pytest pytest-cov
 
 # Remove `-frozen` to build without strictly pinned dependencies.
-COPY requirements-frozen.txt /mist.api/requirements.txt
-COPY requirements-frozen.txt /requirements-frozen-mist.api.txt
+COPY requirements.txt /mist.api/requirements.txt
 COPY requirements.txt /requirements-mist.api.txt
 
 WORKDIR /mist.api/
 
-COPY paramiko /mist.api/paramiko
 COPY lc /mist.api/lc
 COPY v2 /mist.api/v2
 
-RUN pip install --no-cache-dir -r /mist.api/requirements.txt && \
-    pip install -e paramiko/ && \
-    pip install -e lc/ && \
-    pip install -e v2/ && \
-    pip install --no-cache-dir -r v2/requirements.txt
+RUN pip install --no-cache-dir -r /mist.api/requirements.txt
+RUN pip install -e lc/
+RUN pip install -e v2/
+RUN pip install --no-cache-dir -r v2/requirements.txt --config-setting editable_mode=compat
 
 COPY . /mist.api/
 

azure_default_images.json

@@ -1,14 +1,17 @@
 {
-    "OpenLogic:CentOS:7.5:latest": "OpenLogic CentOS 7.5",
-    "CoreOS:CoreOS:Stable:latest": "CoreOS CoreOS Stable",
-    "Debian:debian-10:10:latest": "Debian debian-10 10",
-    "SUSE:openSUSE-Leap:42.3:latest": "SUSE openSUSE-Leap 42.3",
-    "RedHat:RHEL:7-LVM:latest": "RedHat RHEL 7-LVM",
-    "SUSE:SLES:15:latest": "SUSE SLES 15",
-    "Canonical:UbuntuServer:18.04-LTS:latest": "Canonical UbuntuServer 18.04-LTS",
-    "MicrosoftWindowsServer:WindowsServer:2019-Datacenter:latest": "MicrosoftWindowsServer WindowsServer 2019-Datacenter",
-    "MicrosoftWindowsServer:WindowsServer:2016-Datacenter:latest": "MicrosoftWindowsServer WindowsServer 2016-Datacenter",
+    "OpenLogic:CentOS:8_5-gen2:latest": "OpenLogic CentOS 8_5-gen2",
+    "Debian:debian-11:11-backports-gen2:latest": "Debian debian-11 11-backports-gen2",
+    "SUSE:openSUSE-leap-15-4:gen2:latest": "SUSE openSUSE-leap-15-4 gen2",
+    "RedHat:RHEL:8-lvm-gen2:latest": "RedHat RHEL 8-lvm-gen2",
+    "SUSE:sles-15-sp5:gen2:latest": "SUSE sles-15-sp5 gen2",
+    "Canonical:0001-com-ubuntu-server-jammy:22_04-lts-gen2:latest": "Canonical 0001-com-ubuntu-server-jammy 22_04-lts-gen2",
+    "Canonical:ubuntu-24_04-lts:server:latest": "Canonical ubuntu-24_04-lts server",
+    "Canonical:ubuntu-24_04-lts:ubuntu-pro:latest": "Canonical ubuntu-24_04-lts ubuntu-pro",
+    "kinvolk:flatcar-container-linux-free:stable-gen2:latest": "kinvolk flatcar-container-linux-free stable-gen2",
+    "MicrosoftWindowsServer:WindowsServer:2022-datacenter-g2:latest": "MicrosoftWindowsServer WindowsServer 2022-datacenter-g2",
+    "MicrosoftWindowsServer:WindowsServer:2022-datacenter-azure-edition-core:latest": "MicrosoftWindowsServer WindowsServer 2022-datacenter-azure-edition-core",
+    "MicrosoftWindowsServer:WindowsServer:2019-datacenter-gensecond:latest": "MicrosoftWindowsServer WindowsServer 2019-datacenter-gensecond",
+    "MicrosoftWindowsServer:WindowsServer:2016-datacenter-gensecond:latest": "MicrosoftWindowsServer WindowsServer 2016-datacenter-gensecond",
     "MicrosoftWindowsServer:WindowsServer:2012-R2-Datacenter:latest": "MicrosoftWindowsServer WindowsServer 2012-R2-Datacenter",
-    "MicrosoftWindowsServer:WindowsServer:2012-Datacenter:latest": "MicrosoftWindowsServer WindowsServer 2012-Datacenter",
-    "MicrosoftWindowsServer:WindowsServer:2008-R2-SP1:latest": "MicrosoftWindowsServer WindowsServer 2008-R2-SP1"
+    "MicrosoftWindowsServer:WindowsServer:2012-Datacenter:latest": "MicrosoftWindowsServer WindowsServer 2012-Datacenter"
 }

bin/docker-init

@@ -12,7 +12,6 @@ if [ ! -e clean ]; then
     set -x
     unset JS_BUILD
     echo "{\"sha\":\"$VERSION_SHA\",\"name\":\"$VERSION_NAME\",\"repo\":\"$VERSION_REPO\",\"modified\":true}" > /mist-version.json
-    pip install -e $DIR/../paramiko/
     pip install -e $DIR/../lc/
     pip install -e $DIR/../src/
     set +e

bin/get-azure-images

@@ -11,7 +11,7 @@ import requests
 import json
 import argparse
 
-GITHUB_URL = "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/arm-compute/quickstart-templates/aliases.json"  # noqa
+GITHUB_URL = "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/arm-compute/quickstart-templates/aliases.json"  # noqa
 
 
 def parse_args():

requirements.txt

@@ -14,7 +14,7 @@ boto3
 dnspython
 dateparser
 dramatiq
-elasticsearch
+elasticsearch[async]==7.10.1
 flake8
 future
 gevent
@@ -25,11 +25,15 @@ ipython
 iso8601
 jsonpatch
 jsonpickle
-kombu
+kombu==5.4.2
 mongoengine
 mongomock
 names
 netaddr
+
+# Make sure that this matches to paramiko version used by libcloud
+paramiko==3.4.0
+
 parse
 passlib
 pastedeploy  # Required for uwsgi paste logging, can probably be removed.
@@ -38,14 +42,14 @@ pika
 pingparsing
 pretty
 pycryptodome
-pylibmc
 pymongo
 pyyaml
 pyramid
 pyramid_chameleon
 pytest
 python3-openid
 pyvmomi
+redis
 requests
 rstream
 s3cmd

src/mist/api/auth/middleware.py

@@ -40,7 +40,10 @@ def session_start_response(status, headers, exc_info=None):
             if isinstance(session, SessionToken) and \
                     not getattr(session, 'internal', False) and \
                     not session.last_accessed_at:
-                cookie = 'session.id=%s; Path=/;' % session.token
+                # (CSRF) Security Fix: Added SameSite=Strict flag to prevent CSRF attack
+                # in admin's "su" operation. In case of having a cookie issue, consider 
+                # removing and addressing the CSRF issue in another way
+                cookie = 'session.id=%s; Path=/; SameSite=Strict;' % session.token
                 headers.append(('Set-Cookie', cookie))
 
             # ApiTokens with 'dummy' in name are handed out by session from

src/mist/api/auth/views.py

@@ -112,6 +112,8 @@ def create_token(request):
     """
     params = params_from_request(request)
     email = params.get('email', '').lower()
+    # (Account Takeover) Security Fix 1/2
+    password = params.get('password', '')
     api_token_name = params.get('name', '')
     org_id = params.get('org_id', '')
     ttl = params.get('ttl', 60 * 60)
@@ -143,8 +145,12 @@ def create_token(request):
                     pass
         try:
             user = User.objects.get(email=email)
+            # (Account Takeover) Security Fix 2/2: Prevent unauthorized API token creation  
+            # Ensure the authenticated user owns the account before issuing a token 
+            if not user.check_password(password):  
+                raise UserUnauthorizedError("Invalid credentials")
         except User.DoesNotExist:
-            raise UserUnauthorizedError()
+            raise UserUnauthorizedError("Invalid credentials")
         # Remove org is not None when we enforce org context on tokens.
         if org is not None and user not in org.members:
             raise ForbiddenError()