From fe623ea43f75bc2ea546d76bba31f37479d85703 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Sun, 20 Jul 2025 14:16:08 +0100
Subject: [PATCH 01/25] Add conda and pip caching

---
 .github/workflows/integration_tests.yml | 41 +++++++++++++++++++------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 6bfca7f..2ac1c23 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -49,6 +49,27 @@ jobs:
           activate-environment: "test"
           conda-remove-defaults: "true"
 
+      - name: Cache pip
+        uses: actions/cache@v4
+        env:
+          # Increase this value to reset cache
+          CACHE_NUMBER: 0
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip--${{ env.CACHE_NUMBER }}
+          restore-keys: |
+            ${{ runner.os }}-pip-${{ env.CACHE_NUMBER }}
+
+      - name: Cache conda
+        uses: actions/cache@v4
+        env:
+          # Increase this value to reset cache
+          CACHE_NUMBER: 0
+        with:
+          path: ~/conda_pkgs_dir
+          key:
+            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}
+
       - name: Conda info
         run: |
           conda info
@@ -157,7 +178,7 @@ jobs:
         run: |
           conda list
 
-      - name: Install ${{ env.REPOSITORY_NAME }}
+      - name: Install ${{ env.REPOSITORY_NAME }} (local)
         run: |
           pip install -e .
 
@@ -166,46 +187,46 @@ jobs:
           conda list
 
       - name: Run HyperSpy Test Suite
-        if: ${{ env.REPOSITORY_NAME != 'hyperspy' }}
+        if: ${{ always() && env.REPOSITORY_NAME != 'hyperspy' }}
         run: |
           python -m pytest --pyargs hyperspy ${{ env.PYTEST_ARGS }}
 
       - name: Run RosettaSciIO Test Suite
-        if: ${{ env.REPOSITORY_NAME != 'rosettasciio' }}
+        if: ${{ always() && env.REPOSITORY_NAME != 'rosettasciio' }}
         run: |
           python -m pytest --pyargs rsciio ${{ env.PYTEST_ARGS }}
 
       - name: Run LumiSpy Test Suite
-        if: ${{ contains(inputs.EXTENSIONS, 'lumispy') && env.REPOSITORY_NAME != 'lumispy' }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'lumispy') && env.REPOSITORY_NAME != 'lumispy' }}
         run: |
           python -m pytest --pyargs lumispy
 
       - name: Run Pyxem Test Suite
-        if: ${{ contains(inputs.EXTENSIONS, 'pyxem') && env.REPOSITORY_NAME != 'pyxem' }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'pyxem') && env.REPOSITORY_NAME != 'pyxem' }}
         run: |
           python -m pytest --pyargs pyxem
 
       - name: Run holospy Test Suite
-        if: ${{ contains(inputs.EXTENSIONS, 'holospy') && env.REPOSITORY_NAME != 'holospy'  }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'holospy') && env.REPOSITORY_NAME != 'holospy'  }}
         run: |
           python -m pytest --pyargs holospy
 
       - name: Run exSpy Test Suite
-        if: ${{ contains(inputs.EXTENSIONS, 'exspy') && env.REPOSITORY_NAME != 'exspy' }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'exspy') && env.REPOSITORY_NAME != 'exspy' }}
         run: |
           python -m pytest --pyargs exspy
 
       - name: Run kikuchipy Test Suite
-        if: ${{ contains(inputs.EXTENSIONS, 'kikuchipy') && env.REPOSITORY_NAME != 'kikuchipy' }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'kikuchipy') && env.REPOSITORY_NAME != 'kikuchipy' }}
         run: |
           python -m pytest --pyargs kikuchipy
 
       - name: Run etspy Test Suite
-        if: ${{ contains(inputs.EXTENSIONS, 'etspy') && env.REPOSITORY_NAME != 'etspy' }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'etspy') && env.REPOSITORY_NAME != 'etspy' }}
         run: |
           python -m pytest --pyargs etspy
 
       - name: Run atomap Test Suite
-        if: ${{ contains(inputs.EXTENSIONS, 'atomap') && env.REPOSITORY_NAME != 'atomap'   }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'atomap') && env.REPOSITORY_NAME != 'atomap' }}
         run: |
           python -m pytest --pyargs atomap

From 3648e6c9949e5e81c043edcbbc24559d2db5e931 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Sun, 20 Jul 2025 15:44:06 +0100
Subject: [PATCH 02/25] Fix exspy install

---
 .github/workflows/integration_tests.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 2ac1c23..f9b73c1 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -144,7 +144,7 @@ jobs:
               pip install "rosettasciio[all]"
             fi
           elif [[ "${{ inputs.ROSETTASCIIO_VERSION }}" == "dev" ]]; then
-            pip install "rosettasciio[all] @ git+https://github.com/hyperspy/rosettasciio.git@main"
+            pip install "rosettasciio[all] @ git+https://github.com/hyperspy/rosettasciio.git"
           else
             echo "Unknown ROSETTASCIIO_VERSION: ${{ inputs.ROSETTASCIIO_VERSION }}"
             exit 1
@@ -155,19 +155,19 @@ jobs:
         run: |
           for ext in $(echo "${{ inputs.EXTENSIONS }}" | tr ' ' '\n'); do
             if [[ "$ext" == "exspy" && "${{ env.REPOSITORY_NAME }}" != "exspy" ]]; then
-              pip install "https://github.com/hyperspy/exspy/archive/main.zip"
+              pip install "exspy @ git+https://github.com/hyperspy/exspy.git"
             elif [[ "$ext" == "holospy" && "${{ env.REPOSITORY_NAME }}" != "holospy" ]]; then
-              pip install "https://github.com/hyperspy/holospy/archive/main.zip"
+              pip install "holospy @ git+https://github.com/hyperspy/holospy.git"
             elif [[ "$ext" == "lumispy" && "${{ env.REPOSITORY_NAME }}" != "lumispy" ]]; then
-              pip install "https://github.com/lumispy/lumispy/archive/main.zip"
+              pip install "lumispy @ git+https://github.com/lumispy/lumispy.git"
             elif [[ "$ext" == "pyxem" && "${{ env.REPOSITORY_NAME }}" != "pyxem" ]]; then
-              pip install "https://github.com/pyxem/pyxem/archive/main.zip"
+              pip install "pyxem @ git+https://github.com/pyxem/pyxem.git"
             elif [[ "$ext" == "kikuchipy" && "${{ env.REPOSITORY_NAME }}" != "kikuchipy" ]]; then
-              pip install "https://github.com/pyxem/kikuchipy/archive/develop.zip"
+              pip install "kikuchipy @ git+https://github.com/pyxem/kikuchipy.git"
             elif [[ "$ext" == "atomap" && "${{ env.REPOSITORY_NAME }}" != "atomap" ]]; then
-              pip install "https://github.com/atomap-dev/atomap/archive/main.zip"
+              pip install "atomap @ git+https://github.com/atomap-dev/atomap.git"
             elif [[ "$ext" == "etspy" && "${{ env.REPOSITORY_NAME }}" != "etspy" ]]; then
-              pip install "https://github.com/usnistgov/etspy/archive/master.zip"
+              pip install "etspy @ git+https://github.com/usnistgov/etspy.git"
             else
               echo "Unknown extension: $ext"
               exit 1

From 764b7da0e4ee4c2b5a0f7e2a3b75bb238356e0ea Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Sun, 20 Jul 2025 17:38:41 +0100
Subject: [PATCH 03/25] Fix etspy and atomap test suite

---
 .github/workflows/integration_tests.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index f9b73c1..5f03133 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -32,7 +32,7 @@ jobs:
         shell: bash -el {0}
     env:
       MPLBACKEND: agg
-      TEST_DEPS: pytest pytest-xdist pytest-rerunfailures pytest-instafail pytest-mpl
+      TEST_DEPS: pytest pytest-xdist pytest-rerunfailures pytest-instafail pytest-mpl filelock
       PYTHON_VERSION: '3.12'
 
     steps:
@@ -100,8 +100,9 @@ jobs:
       - name: Install astra-toolbox, ffmpeg and test dependencies
         if: ${{ contains(inputs.EXTENSIONS, 'etspy') }}
         run: |
-          # astra-toolbox and ffmpeg are etspy dependencies
-          conda install astra-toolbox
+          # astra-toolbox is an etspy dependency
+          # install ffmpeg until test is skipped if not installed
+          conda install astra-toolbox ffmpeg
 
       - name: Install extensions (release)
         if: ${{ contains(inputs.EXTENSION_VERSION, 'release') }}
@@ -229,4 +230,4 @@ jobs:
       - name: Run atomap Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'atomap') && env.REPOSITORY_NAME != 'atomap' }}
         run: |
-          python -m pytest --pyargs atomap
+          python -m pytest --pyargs atomap -k "not test_gui_function_qt"

From d08566237096d230adad52d0660257a05b3d7f57 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Sun, 20 Jul 2025 17:39:04 +0100
Subject: [PATCH 04/25] Add option to install dependencies pre-release or
 development version

---
 .github/workflows/integration_tests.yml | 31 ++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 5f03133..91c2cbf 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -23,6 +23,18 @@ on:
         description: 'Whether to use conda to install the packages. Set to "false" to use pip.'
         default: false
         type: boolean
+      NUMBA_DEV:
+        description: 'Whether to install the numba development version.'
+        default: false
+        type: boolean
+      DEPENDENCIES_PRE_RELEASE:
+        description: 'List of dependencies to install pre-release installed with packages from pypi.org.'
+        default: '' # e.g. matplotlib scipy scikit-learn sympy h5py scikit-image numba
+        type: string
+      DEPENDENCIES_DEV:
+        description: 'List of dependencies to install from scientific-python-nightly-wheels.'
+        default: '' # e.g. matplotlib scipy scikit-learn sympy h5py scikit-image 
+        type: string
 
 jobs:
   integration_test:
@@ -79,6 +91,23 @@ jobs:
         run: |
           conda install pip ${{ env.TEST_DEPS }} 
 
+      - name: Install numba development version
+        if: ${{ inputs.NUMBA_DEV }}
+        run: |
+          mamba install -c numba/label/dev numba
+
+      - name: Install dependencies pre-release version
+        if: ${{ inputs.DEPENDENCIES_PRE_RELEASE }}
+        run: |
+          pip install --upgrade --pre ${{ inputs.DEPENDENCIES_PRE_RELEASE }}
+
+      - name: Install dependencies development version
+        if: ${{ inputs.DEPENDENCIES_DEV }}
+        run: |
+          pip install --upgrade --pre --extra-index-url \
+          https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
+          ${{ inputs.DEPENDENCIES_DEV }}
+
       - name: Get the number of CPUs
         id: cpus
         run: |
@@ -97,7 +126,7 @@ jobs:
           PYTEST_ARGS="--reruns 3 -n ${{ steps.cpus.outputs.count }}"
           echo "PYTEST_ARGS=${PYTEST_ARGS}" >> $GITHUB_ENV
 
-      - name: Install astra-toolbox, ffmpeg and test dependencies
+      - name: Install astra-toolbox and test dependencies
         if: ${{ contains(inputs.EXTENSIONS, 'etspy') }}
         run: |
           # astra-toolbox is an etspy dependency

From 97baabe780e1bf8d05f74821176a1e709f1412eb Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Sun, 20 Jul 2025 17:51:19 +0100
Subject: [PATCH 05/25] Add INSTALL_SOURCE_FROM_REPOSITORY option

---
 .github/workflows/integration_tests.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 91c2cbf..5378bd4 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -19,6 +19,10 @@ on:
         description: 'The version of the rosettasciio to test. It can be "release" or "dev".'
         default: 'release'
         type: string
+      INSTALL_SOURCE_FROM_REPOSITORY:
+        description: 'Whether to install the source code of the current repository.'
+        default: true
+        type: boolean
       USE_CONDA:
         description: 'Whether to use conda to install the packages. Set to "false" to use pip.'
         default: false
@@ -32,6 +36,7 @@ on:
         default: '' # e.g. matplotlib scipy scikit-learn sympy h5py scikit-image numba
         type: string
       DEPENDENCIES_DEV:
+        # Dev version from https://anaconda.org/scientific-python-nightly-wheels/
         description: 'List of dependencies to install from scientific-python-nightly-wheels.'
         default: '' # e.g. matplotlib scipy scikit-learn sympy h5py scikit-image 
         type: string
@@ -209,6 +214,7 @@ jobs:
           conda list
 
       - name: Install ${{ env.REPOSITORY_NAME }} (local)
+        if: ${{ inputs.INSTALL_SOURCE_FROM_REPOSITORY }}
         run: |
           pip install -e .
 

From 7a196ce14cebf227bc5236d5c72ddcfd99eedb20 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Sun, 20 Jul 2025 18:28:20 +0100
Subject: [PATCH 06/25] Add PLATFORM parameter to integration_tests.yml

---
 .github/workflows/integration_tests.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 5378bd4..7b7d630 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -40,10 +40,14 @@ on:
         description: 'List of dependencies to install from scientific-python-nightly-wheels.'
         default: '' # e.g. matplotlib scipy scikit-learn sympy h5py scikit-image 
         type: string
+      PLATFORM:
+        description: 'The platform to run the tests on.'
+        default: 'ubuntu-latest'
+        type: string
 
 jobs:
   integration_test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.PLATFORM }}
     defaults:
       run:
         shell: bash -el {0}

From a9ea598069047a42e634659d1d0a34cad4219fe1 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Sun, 20 Jul 2025 20:07:41 +0100
Subject: [PATCH 07/25] Skip mrcz and usid test since they don't support numpy
 2

---
 .github/workflows/integration_tests.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 7b7d630..3411137 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -234,7 +234,8 @@ jobs:
       - name: Run RosettaSciIO Test Suite
         if: ${{ always() && env.REPOSITORY_NAME != 'rosettasciio' }}
         run: |
-          python -m pytest --pyargs rsciio ${{ env.PYTEST_ARGS }}
+          # mrcz and pyusid doesn't support numpy 2
+          python -m pytest --pyargs rsciio ${{ env.PYTEST_ARGS }} -k "not test_mrcz.py and not test_usid.py"
 
       - name: Run LumiSpy Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'lumispy') && env.REPOSITORY_NAME != 'lumispy' }}

From f7fb3894d1afb5baf5cae47c0e3da9ec1e22155b Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Sun, 20 Jul 2025 20:11:58 +0100
Subject: [PATCH 08/25] Always run testing when library is installed

---
 .github/workflows/integration_tests.yml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 3411137..a237f79 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -227,47 +227,47 @@ jobs:
           conda list
 
       - name: Run HyperSpy Test Suite
-        if: ${{ always() && env.REPOSITORY_NAME != 'hyperspy' }}
+        if: ${{ always() }}
         run: |
           python -m pytest --pyargs hyperspy ${{ env.PYTEST_ARGS }}
 
       - name: Run RosettaSciIO Test Suite
-        if: ${{ always() && env.REPOSITORY_NAME != 'rosettasciio' }}
+        if: ${{ always() }}
         run: |
           # mrcz and pyusid doesn't support numpy 2
           python -m pytest --pyargs rsciio ${{ env.PYTEST_ARGS }} -k "not test_mrcz.py and not test_usid.py"
 
       - name: Run LumiSpy Test Suite
-        if: ${{ always() && contains(inputs.EXTENSIONS, 'lumispy') && env.REPOSITORY_NAME != 'lumispy' }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'lumispy')}}
         run: |
           python -m pytest --pyargs lumispy
 
       - name: Run Pyxem Test Suite
-        if: ${{ always() && contains(inputs.EXTENSIONS, 'pyxem') && env.REPOSITORY_NAME != 'pyxem' }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'pyxem') }}
         run: |
           python -m pytest --pyargs pyxem
 
       - name: Run holospy Test Suite
-        if: ${{ always() && contains(inputs.EXTENSIONS, 'holospy') && env.REPOSITORY_NAME != 'holospy'  }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'holospy') }}
         run: |
           python -m pytest --pyargs holospy
 
       - name: Run exSpy Test Suite
-        if: ${{ always() && contains(inputs.EXTENSIONS, 'exspy') && env.REPOSITORY_NAME != 'exspy' }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'exspy') }}
         run: |
           python -m pytest --pyargs exspy
 
       - name: Run kikuchipy Test Suite
-        if: ${{ always() && contains(inputs.EXTENSIONS, 'kikuchipy') && env.REPOSITORY_NAME != 'kikuchipy' }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'kikuchipy') }}
         run: |
           python -m pytest --pyargs kikuchipy
 
       - name: Run etspy Test Suite
-        if: ${{ always() && contains(inputs.EXTENSIONS, 'etspy') && env.REPOSITORY_NAME != 'etspy' }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'etspy') }}
         run: |
           python -m pytest --pyargs etspy
 
       - name: Run atomap Test Suite
-        if: ${{ always() && contains(inputs.EXTENSIONS, 'atomap') && env.REPOSITORY_NAME != 'atomap' }}
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'atomap') }}
         run: |
           python -m pytest --pyargs atomap -k "not test_gui_function_qt"

From 3045e0b8aa0cf241fd860d216415d255cc1eda5a Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Sun, 20 Jul 2025 20:13:59 +0100
Subject: [PATCH 09/25] Fetch tag to get correct version

---
 .github/workflows/integration_tests.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index a237f79..a5dd3fa 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -58,11 +58,22 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with: 
+          fetch-depth: 0
+          fetch-tags: true
 
       - name: Get repository name
         run: |
           echo "REPOSITORY_NAME=${GITHUB_REPOSITORY#*/}" >> $GITHUB_ENV
 
+      - name: Fetch tags upstream
+        if: ${{ github.repository_owner != 'hyperspy' }}
+        # Needs to fetch the tags from upstream to get the
+        # correct version with setuptools_scm
+        run: |
+          git remote add upstream https://github.com/hyperspy/${{ env.REPOSITORY_NAME }}.git
+          git fetch upstream --tags
+
       - uses: conda-incubator/setup-miniconda@v3
         with:
           miniforge-version: latest

From 76a980e7e235d9558c5cd56df55cdc1d67e1a4e5 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Mon, 21 Jul 2025 08:35:14 +0100
Subject: [PATCH 10/25] Add pooch cache

---
 .github/workflows/integration_tests.yml | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index a5dd3fa..0d42249 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -81,6 +81,14 @@ jobs:
           activate-environment: "test"
           conda-remove-defaults: "true"
 
+      - name: Install pip and test dependencies
+        run: |
+          conda install pip pooch ${{ env.TEST_DEPS }} 
+
+      - name: Get pooch cache location
+        run: |
+          python -c "import pooch; print(f'POOCH_CACHE={pooch.os_cache()}', end='')" >> $GITHUB_ENV
+
       - name: Cache pip
         uses: actions/cache@v4
         env:
@@ -102,15 +110,21 @@ jobs:
           key:
             ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}
 
+      - name: Cache pooch
+        uses: actions/cache@v4
+        env:
+          # Increase this value to reset cache
+          CACHE_NUMBER: 0
+        with:
+          path: ${{ env.POOCH_CACHE }}
+          key:
+            ${{ runner.os }}-pooch-${{ env.CACHE_NUMBER }}
+
       - name: Conda info
         run: |
           conda info
           conda list
 
-      - name: Install pip and test dependencies
-        run: |
-          conda install pip ${{ env.TEST_DEPS }} 
-
       - name: Install numba development version
         if: ${{ inputs.NUMBA_DEV }}
         run: |

From 9a267413f667b284ac0bb3f62891361a254c1e53 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Mon, 21 Jul 2025 08:51:48 +0100
Subject: [PATCH 11/25] Add `ADDITIONAL_PACKAGES` parameter to allow
 installation of additional packages

---
 .github/workflows/integration_tests.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 0d42249..89890e0 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -27,6 +27,10 @@ on:
         description: 'Whether to use conda to install the packages. Set to "false" to use pip.'
         default: false
         type: boolean
+      ADDITIONAL_PACKAGES:
+        description: 'List of additional packages to install.'
+        default: ''
+        type: string
       NUMBA_DEV:
         description: 'Whether to install the numba development version.'
         default: false
@@ -125,6 +129,15 @@ jobs:
           conda info
           conda list
 
+      - name: Install additional packages
+        if: ${{ inputs.ADDITIONAL_PACKAGES != '' }}
+        run: |
+          if [[ "${{ inputs.USE_CONDA }}" == "true" ]]; then
+            conda install ${{ inputs.ADDITIONAL_PACKAGES }}
+          else
+            pip install ${{ inputs.ADDITIONAL_PACKAGES }}
+          fi
+
       - name: Install numba development version
         if: ${{ inputs.NUMBA_DEV }}
         run: |

From 371bc20e93afe1e169f183e1145bacbb0e1553c8 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Mon, 21 Jul 2025 09:15:47 +0100
Subject: [PATCH 12/25] Add cache for rosettasciio test data

---
 .github/workflows/integration_tests.yml | 32 +++++++++++++------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 89890e0..7590285 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -87,11 +87,7 @@ jobs:
 
       - name: Install pip and test dependencies
         run: |
-          conda install pip pooch ${{ env.TEST_DEPS }} 
-
-      - name: Get pooch cache location
-        run: |
-          python -c "import pooch; print(f'POOCH_CACHE={pooch.os_cache()}', end='')" >> $GITHUB_ENV
+          conda install pip ${{ env.TEST_DEPS }} 
 
       - name: Cache pip
         uses: actions/cache@v4
@@ -114,16 +110,6 @@ jobs:
           key:
             ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}
 
-      - name: Cache pooch
-        uses: actions/cache@v4
-        env:
-          # Increase this value to reset cache
-          CACHE_NUMBER: 0
-        with:
-          path: ${{ env.POOCH_CACHE }}
-          key:
-            ${{ runner.os }}-pooch-${{ env.CACHE_NUMBER }}
-
       - name: Conda info
         run: |
           conda info
@@ -264,6 +250,22 @@ jobs:
         run: |
           conda list
 
+      - name: Get rosettasciio tests data location
+        if: ${{ env.REPOSITORY_NAME != 'rosettasciio' }} # only for repositories that are not rosettasciio
+        run: |
+          python -c "from rsciio.tests import registry; print(f'ROSETTASCIIO_TEST_DATA={str(registry.TESTS_PATH / \"data\") }')" >> $GITHUB_ENV
+
+      - name: Cache rosettasciio data
+        if: ${{ env.REPOSITORY_NAME != 'rosettasciio' }} # only for repositories that are not rosettasciio
+        uses: actions/cache@v4
+        env:
+          # Increase this value to reset cache
+          CACHE_NUMBER: 0
+        with:
+          path: ${{ env.ROSETTASCIIO_TEST_DATA }}
+          key:
+            ${{ runner.os }}-rosettasciio-${{ env.CACHE_NUMBER }}
+
       - name: Run HyperSpy Test Suite
         if: ${{ always() }}
         run: |

From e8fb6f401c7b2c3cecc0d1f6b51df3d1c0fcbc68 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Mon, 21 Jul 2025 09:53:06 +0100
Subject: [PATCH 13/25] Add `PIP_EXTRAS` parameters to specify pip extras when
 installing source from current repository

---
 .github/workflows/integration_tests.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 7590285..c56fd6e 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -48,6 +48,11 @@ on:
         description: 'The platform to run the tests on.'
         default: 'ubuntu-latest'
         type: string
+      PIP_EXTRAS:
+        description: 'Optional “variants” (also called extra) to be installed for the source of the current repository'
+        # if empty string, no argument is passed
+        default: ''
+        type: string
 
 jobs:
   integration_test:
@@ -244,7 +249,7 @@ jobs:
       - name: Install ${{ env.REPOSITORY_NAME }} (local)
         if: ${{ inputs.INSTALL_SOURCE_FROM_REPOSITORY }}
         run: |
-          pip install -e .
+          pip install -e .${{ inputs.PIP_EXTRAS }}
 
       - name: Conda list
         run: |

From 3f0911ceb19576c0ac16e241a46e952e5182610d Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Mon, 21 Jul 2025 11:08:44 +0100
Subject: [PATCH 14/25] Use different atomap branch as a workaround for test
 data packaging issue

---
 .github/workflows/integration_tests.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index c56fd6e..3db1718 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -233,7 +233,9 @@ jobs:
             elif [[ "$ext" == "kikuchipy" && "${{ env.REPOSITORY_NAME }}" != "kikuchipy" ]]; then
               pip install "kikuchipy @ git+https://github.com/pyxem/kikuchipy.git"
             elif [[ "$ext" == "atomap" && "${{ env.REPOSITORY_NAME }}" != "atomap" ]]; then
-              pip install "atomap @ git+https://github.com/atomap-dev/atomap.git"
+              # pip install "atomap @ git+https://github.com/atomap-dev/atomap.git"
+              # https://github.com/atomap-dev/atomap/pull/2
+              pip install "atomap @ git+https://github.com/ericpre/atomap.git@fix_packaging"
             elif [[ "$ext" == "etspy" && "${{ env.REPOSITORY_NAME }}" != "etspy" ]]; then
               pip install "etspy @ git+https://github.com/usnistgov/etspy.git"
             else

From 0695d2684962f62399714aa3c677f83a37a21e6b Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Mon, 21 Jul 2025 13:56:04 +0100
Subject: [PATCH 15/25] Use different kikuchipy branch as a workaround for
 allow_download issue in the test suite

---
 .github/workflows/integration_tests.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 3db1718..e443ae6 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -231,7 +231,9 @@ jobs:
             elif [[ "$ext" == "pyxem" && "${{ env.REPOSITORY_NAME }}" != "pyxem" ]]; then
               pip install "pyxem @ git+https://github.com/pyxem/pyxem.git"
             elif [[ "$ext" == "kikuchipy" && "${{ env.REPOSITORY_NAME }}" != "kikuchipy" ]]; then
-              pip install "kikuchipy @ git+https://github.com/pyxem/kikuchipy.git"
+              # pip install "kikuchipy @ git+https://github.com/pyxem/kikuchipy.git"
+              # fix allow_download issue
+              pip install "kikuchipy @ git+https://github.com/ericpre/kikuchipy.git@fix_allow_download_test_suite"
             elif [[ "$ext" == "atomap" && "${{ env.REPOSITORY_NAME }}" != "atomap" ]]; then
               # pip install "atomap @ git+https://github.com/atomap-dev/atomap.git"
               # https://github.com/atomap-dev/atomap/pull/2
@@ -307,7 +309,8 @@ jobs:
       - name: Run kikuchipy Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'kikuchipy') }}
         run: |
-          python -m pytest --pyargs kikuchipy
+          # https://github.com/pyxem/kikuchipy/issues/707
+          python -m pytest --pyargs kikuchipy -k "not test_not_allow_download_raises"
 
       - name: Run etspy Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'etspy') }}

From 9b7303beb2969f2501531146c7be0c07b2ed363f Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Mon, 21 Jul 2025 14:21:47 +0100
Subject: [PATCH 16/25] Skip kikuchipy test that fails because of bugs in the
 test suite

---
 .github/workflows/integration_tests.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index e443ae6..0648f1c 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -232,7 +232,7 @@ jobs:
               pip install "pyxem @ git+https://github.com/pyxem/pyxem.git"
             elif [[ "$ext" == "kikuchipy" && "${{ env.REPOSITORY_NAME }}" != "kikuchipy" ]]; then
               # pip install "kikuchipy @ git+https://github.com/pyxem/kikuchipy.git"
-              # fix allow_download issue
+              # https://github.com/pyxem/kikuchipy/pull/741
               pip install "kikuchipy @ git+https://github.com/ericpre/kikuchipy.git@fix_allow_download_test_suite"
             elif [[ "$ext" == "atomap" && "${{ env.REPOSITORY_NAME }}" != "atomap" ]]; then
               # pip install "atomap @ git+https://github.com/atomap-dev/atomap.git"
@@ -310,7 +310,8 @@ jobs:
         if: ${{ always() && contains(inputs.EXTENSIONS, 'kikuchipy') }}
         run: |
           # https://github.com/pyxem/kikuchipy/issues/707
-          python -m pytest --pyargs kikuchipy -k "not test_not_allow_download_raises"
+          # https://github.com/pyxem/kikuchipy/pull/741
+          python -m pytest --pyargs kikuchipy -k "not test_not_allow_download_raises and not test_ebsd"
 
       - name: Run etspy Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'etspy') }}

From 23e56e5689a429919775f89dd753b25b325469d3 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Mon, 21 Jul 2025 14:50:41 +0100
Subject: [PATCH 17/25] Clean up `package_and_tests.yml` workflow

---
 .github/workflows/package_and_test.yml | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/package_and_test.yml b/.github/workflows/package_and_test.yml
index d787a6b..382e095 100644
--- a/.github/workflows/package_and_test.yml
+++ b/.github/workflows/package_and_test.yml
@@ -155,12 +155,8 @@ jobs:
           fi
 
       - name: Install distribution
-        env:
-          EXTRAS: ${{ inputs.EXTRAS }}
         run: |
-          # pint release version doesn't support numpy 2.0
-          pip install numpy
-          pip install --pre --find-links dist ${{ env.LIBRARY_NAME }}${{ env.EXTRAS }}
+          pip install --pre --find-links dist ${{ env.LIBRARY_NAME }}${{ inputs.EXTRAS }}
 
       - name: Install additional test dependencies
         if: ${{ inputs.ADDITIONAL_TEST_DEPENDENCIES != ''}}
@@ -175,15 +171,13 @@ jobs:
         if: ${{ inputs.use_xvfb != true }}
         env:
           POOCH_BASE_URL: ${{ inputs.POOCH_BASE_URL }}
-          PYTEST_ARGS: ${{ inputs.PYTEST_ARGS }}
         run: |
-          pytest --pyargs ${{ env.MODULE_NAME }} ${{ env.PYTEST_ARGS }}
+          pytest --pyargs ${{ env.MODULE_NAME }} ${{ inputs.PYTEST_ARGS }}
 
       - name: Test distribution (with xvfb)
         if: ${{ inputs.use_xvfb }}
         env:
           POOCH_BASE_URL: ${{ inputs.POOCH_BASE_URL }}
-          PYTEST_ARGS: ${{ inputs.PYTEST_ARGS }}
         run: |
           sudo apt-get install xvfb
-          xvfb-run pytest --pyargs ${{ env.MODULE_NAME }} ${{ env.PYTEST_ARGS }}
+          xvfb-run pytest --pyargs ${{ env.MODULE_NAME }} ${{ inputs.PYTEST_ARGS }}

From 522d71e829321a53677b43f0914aded6e5c8ef25 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Mon, 21 Jul 2025 15:36:55 +0100
Subject: [PATCH 18/25] Add gui packages

---
 .github/workflows/integration_tests.yml | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 0648f1c..c09f512 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -5,7 +5,7 @@ on:
     inputs:
       EXTENSIONS:
         description: 'The list of extension to test.'
-        default: 'exspy holospy lumispy'
+        default: 'exspy holospy lumispy hyperspy-gui-ipywidgets hyperspy-gui-traitsui'
         type: string
       EXTENSION_VERSION:
         description: 'The version of the extension to test. It can be "release" or "dev".'
@@ -226,6 +226,10 @@ jobs:
               pip install "exspy @ git+https://github.com/hyperspy/exspy.git"
             elif [[ "$ext" == "holospy" && "${{ env.REPOSITORY_NAME }}" != "holospy" ]]; then
               pip install "holospy @ git+https://github.com/hyperspy/holospy.git"
+            elif [[ "$ext" == "hyperspy-gui-ipywidgets" && "${{ env.REPOSITORY_NAME }}" != "hyperspy_gui_ipywidgets" ]]; then
+              pip install "hyperspy-gui-ipywidgets @ git+https://github.com/hyperspy/hyperspy_gui_ipywidgets.git"
+            elif [[ "$ext" == "hyperspy-gui-traitsui" && "${{ env.REPOSITORY_NAME }}" != "hyperspy_gui_traitsui" ]]; then
+              pip install "hyperspy-gui-traitsui @ git+https://github.com/hyperspy/hyperspy_gui_traitsui.git"
             elif [[ "$ext" == "lumispy" && "${{ env.REPOSITORY_NAME }}" != "lumispy" ]]; then
               pip install "lumispy @ git+https://github.com/lumispy/lumispy.git"
             elif [[ "$ext" == "pyxem" && "${{ env.REPOSITORY_NAME }}" != "pyxem" ]]; then
@@ -291,6 +295,16 @@ jobs:
         run: |
           python -m pytest --pyargs lumispy
 
+      - name: Run hyperspy_gui_ipywidgets Test Suite
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'hyperspy-gui-ipywidgets') }}
+        run: |
+          python -m pytest --pyargs hyperspy_gui_ipywidgets
+
+      - name: Run hyperspy_gui_traitsui Test Suite
+        if: ${{ always() && contains(inputs.EXTENSIONS, 'hyperspy-gui-traitsui') }}
+        run: |
+          python -m pytest --pyargs hyperspy_gui_traitsui
+
       - name: Run Pyxem Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'pyxem') }}
         run: |

From 9e017d91139943408fe7a5c85d12f2562f56c3fc Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Mon, 21 Jul 2025 15:43:16 +0100
Subject: [PATCH 19/25] Install optional dependencies

---
 .github/workflows/integration_tests.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index c09f512..8c6372b 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -175,9 +175,9 @@ jobs:
         if: ${{ contains(inputs.EXTENSION_VERSION, 'release') }}
         run: |
           if [[ "${{ inputs.USE_CONDA }}" == "true" ]]; then
-            conda install hyperspy-base ${{ inputs.EXTENSIONS }}
+            conda install ${{ inputs.EXTENSIONS }}
           else
-            pip install hyperspy[all] ${{ inputs.EXTENSIONS }}
+            pip install ${{ inputs.EXTENSIONS }}
           fi
 
       - name: Conda list
@@ -189,7 +189,7 @@ jobs:
         run: |
           if [[ "${{ inputs.HYPERSPY_VERSION }}" == "release" ]]; then
             if [[ "${{ inputs.USE_CONDA }}" == "true" ]]; then
-              conda install hyperspy-base
+              conda install hyperspy
             else
               pip install "hyperspy[all]"
             fi
@@ -207,7 +207,7 @@ jobs:
         run: |
           if [[ "${{ inputs.ROSETTASCIIO_VERSION }}" == "release" ]]; then
             if [[ "${{ inputs.USE_CONDA }}" == "true" ]]; then
-              conda install rosettasciio-base
+              conda install rosettasciio
             else
               pip install "rosettasciio[all]"
             fi

From 6b21dd7deba3956afc736099ab2c33d43fa72370 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Tue, 22 Jul 2025 12:30:35 +0100
Subject: [PATCH 20/25] Set different default directory to avoid interference
 with repository source code

---
 .github/workflows/integration_tests.yml | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 8c6372b..fa1f586 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -64,7 +64,6 @@ jobs:
       MPLBACKEND: agg
       TEST_DEPS: pytest pytest-xdist pytest-rerunfailures pytest-instafail pytest-mpl filelock
       PYTHON_VERSION: '3.12'
-
     steps:
       - uses: actions/checkout@v4
         with: 
@@ -77,8 +76,6 @@ jobs:
 
       - name: Fetch tags upstream
         if: ${{ github.repository_owner != 'hyperspy' }}
-        # Needs to fetch the tags from upstream to get the
-        # correct version with setuptools_scm
         run: |
           git remote add upstream https://github.com/hyperspy/${{ env.REPOSITORY_NAME }}.git
           git fetch upstream --tags
@@ -235,9 +232,7 @@ jobs:
             elif [[ "$ext" == "pyxem" && "${{ env.REPOSITORY_NAME }}" != "pyxem" ]]; then
               pip install "pyxem @ git+https://github.com/pyxem/pyxem.git"
             elif [[ "$ext" == "kikuchipy" && "${{ env.REPOSITORY_NAME }}" != "kikuchipy" ]]; then
-              # pip install "kikuchipy @ git+https://github.com/pyxem/kikuchipy.git"
-              # https://github.com/pyxem/kikuchipy/pull/741
-              pip install "kikuchipy @ git+https://github.com/ericpre/kikuchipy.git@fix_allow_download_test_suite"
+              pip install "kikuchipy @ git+https://github.com/pyxem/kikuchipy.git"
             elif [[ "$ext" == "atomap" && "${{ env.REPOSITORY_NAME }}" != "atomap" ]]; then
               # pip install "atomap @ git+https://github.com/atomap-dev/atomap.git"
               # https://github.com/atomap-dev/atomap/pull/2
@@ -256,6 +251,7 @@ jobs:
 
       - name: Install ${{ env.REPOSITORY_NAME }} (local)
         if: ${{ inputs.INSTALL_SOURCE_FROM_REPOSITORY }}
+        working-directory: ${{ github.workspace }}
         run: |
           pip install -e .${{ inputs.PIP_EXTRAS }}
 
@@ -281,58 +277,68 @@ jobs:
 
       - name: Run HyperSpy Test Suite
         if: ${{ always() }}
+        working-directory: ${{ runner.temp }}
         run: |
           python -m pytest --pyargs hyperspy ${{ env.PYTEST_ARGS }}
 
       - name: Run RosettaSciIO Test Suite
         if: ${{ always() }}
+        working-directory: ${{ runner.temp }}
         run: |
           # mrcz and pyusid doesn't support numpy 2
           python -m pytest --pyargs rsciio ${{ env.PYTEST_ARGS }} -k "not test_mrcz.py and not test_usid.py"
 
       - name: Run LumiSpy Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'lumispy')}}
+        working-directory: ${{ runner.temp }}
         run: |
           python -m pytest --pyargs lumispy
 
       - name: Run hyperspy_gui_ipywidgets Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'hyperspy-gui-ipywidgets') }}
+        working-directory: ${{ runner.temp }}
         run: |
           python -m pytest --pyargs hyperspy_gui_ipywidgets
 
       - name: Run hyperspy_gui_traitsui Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'hyperspy-gui-traitsui') }}
+        working-directory: ${{ runner.temp }}
         run: |
           python -m pytest --pyargs hyperspy_gui_traitsui
 
       - name: Run Pyxem Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'pyxem') }}
+        working-directory: ${{ runner.temp }}
         run: |
           python -m pytest --pyargs pyxem
 
       - name: Run holospy Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'holospy') }}
+        working-directory: ${{ runner.temp }}
         run: |
           python -m pytest --pyargs holospy
 
       - name: Run exSpy Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'exspy') }}
+        working-directory: ${{ runner.temp }}
         run: |
           python -m pytest --pyargs exspy
 
       - name: Run kikuchipy Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'kikuchipy') }}
+        working-directory: ${{ runner.temp }}
         run: |
           # https://github.com/pyxem/kikuchipy/issues/707
-          # https://github.com/pyxem/kikuchipy/pull/741
-          python -m pytest --pyargs kikuchipy -k "not test_not_allow_download_raises and not test_ebsd"
+          python -m pytest --pyargs kikuchipy -k "not test_not_allow_download_raises"
 
       - name: Run etspy Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'etspy') }}
+        working-directory: ${{ runner.temp }}
         run: |
           python -m pytest --pyargs etspy
 
       - name: Run atomap Test Suite
         if: ${{ always() && contains(inputs.EXTENSIONS, 'atomap') }}
+        working-directory: ${{ runner.temp }}
         run: |
           python -m pytest --pyargs atomap -k "not test_gui_function_qt"

From 9c17b0e9d718cdfe131f6d1abf598eb5c607d11c Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Fri, 25 Jul 2025 09:15:50 +0100
Subject: [PATCH 21/25] Add pooch cache in integration tests

---
 .github/workflows/integration_tests.yml | 31 ++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index fa1f586..ad2a4ae 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -53,6 +53,10 @@ on:
         # if empty string, no argument is passed
         default: ''
         type: string
+      CACHE_POOCH:
+        description: 'The list of pooch cache to cache - see pooch.os_cache() for more information.'
+        default: 'kikuchipy pyxem'
+        type: string
 
 jobs:
   integration_test:
@@ -98,9 +102,9 @@ jobs:
           CACHE_NUMBER: 0
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip--${{ env.CACHE_NUMBER }}
+          key: ${{ runner.os }}-integration-pip-${{ env.CACHE_NUMBER }}
           restore-keys: |
-            ${{ runner.os }}-pip-${{ env.CACHE_NUMBER }}
+            ${{ runner.os }}-integration-pip-${{ env.CACHE_NUMBER }}
 
       - name: Cache conda
         uses: actions/cache@v4
@@ -110,7 +114,9 @@ jobs:
         with:
           path: ~/conda_pkgs_dir
           key:
-            ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}
+            ${{ runner.os }}-integration-conda-${{ env.CACHE_NUMBER }}
+          restore-keys: |
+            ${{ runner.os }}-integration-conda-${{ env.CACHE_NUMBER }}
 
       - name: Conda info
         run: |
@@ -255,6 +261,25 @@ jobs:
         run: |
           pip install -e .${{ inputs.PIP_EXTRAS }}
 
+      - name: Get pooch cache paths
+        if: ${{ inputs.CACHE_POOCH != '' }}
+        run: |
+          # create list of pooch cache paths as a string (space separated)
+          python -c "import pooch; print(' '.join([str(pooch.os_cache(s)) for s in '${{ inputs.CACHE_POOCH }}'.split(' ')]))" > pooch_cache_paths.txt
+          echo "POOCH_CACHE_PATHS=$(cat pooch_cache_paths.txt)" >> $GITHUB_ENV
+
+      - name: Cache pooch data
+        if: ${{ inputs.CACHE_POOCH != '' }}
+        uses: actions/cache@v4
+        env:
+          # Increase this value to reset cache
+          CACHE_NUMBER: 0
+        with:
+          path: ${{ env.POOCH_CACHE_PATHS }}
+          key: ${{ runner.os }}-integration-pooch-${{ env.CACHE_NUMBER }}
+          restore-keys: |
+            ${{ runner.os }}-integration-pooch-${{ env.CACHE_NUMBER }}
+
       - name: Conda list
         run: |
           conda list

From fbaf4e35c8a79d0ba7906540857057535e992692 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Wed, 23 Jul 2025 15:23:43 +0100
Subject: [PATCH 22/25] Add github deploy preview: not public yet and to be
 tested

---
 .github/workflows/push_doc_preview.yml | 28 ++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 .github/workflows/push_doc_preview.yml

diff --git a/.github/workflows/push_doc_preview.yml b/.github/workflows/push_doc_preview.yml
new file mode 100644
index 0000000..8d0e2e7
--- /dev/null
+++ b/.github/workflows/push_doc_preview.yml
@@ -0,0 +1,28 @@
+name: Push Documentation (Preview)
+# preview feature of actions/deploy-pages not available publicly yet
+
+on:
+  workflow_call:
+    inputs:
+      artefact_name:
+        description: 'The name of the github artefact containing the doc'
+        # if empty string, the current repository name is used
+        default: 'doc_html'
+        type: string
+
+jobs:
+  deploy:
+    name: "Pages Preview"
+    runs-on: ubuntu-latest
+    permissions:
+      # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+      contents: read
+      pages: write
+      id-token: write
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
+        with:
+          artifact_name: ${{ inputs.artefact_name }}
+          preview: "true"

From 852792dd2f8941ffac033afe4fdabca188d6f2f8 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Fri, 25 Jul 2025 08:36:36 +0100
Subject: [PATCH 23/25] Add cache documentation build by examples folder and
 pooch cache

---
 .github/workflows/doc.yml | 70 +++++++++++++++++++++++++++++++++++++--
 1 file changed, 67 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml
index 06efd63..8eadb26 100644
--- a/.github/workflows/doc.yml
+++ b/.github/workflows/doc.yml
@@ -44,6 +44,17 @@ on:
         description: 'The organisation hosting the upstream code'
         default: 'hyperspy'
         type: string  
+      CACHE_GALLERY_EXAMPLES:
+        description: 'The path to cache for the documentation build. Useful to speed up documentation build.'
+        default: ''  # './doc/auto_examples'
+        type: string  
+      CACHE_POOCH:
+        description: 'The name of the library to cache the pooch data, see pooch.os_cache() for more details. 
+                      The default pooch cache (pooch.os_cache("pooch")) will also be cached.
+                      If empty (default), no caching is performed.'
+        default: ''
+        type: string  
+
 
 jobs:
   build:
@@ -71,6 +82,18 @@ jobs:
           git remote add upstream https://github.com/${{ inputs.ORGANISATION }}/${{ env.REPOSITORY_NAME }}.git
           git fetch upstream --tags
 
+      - name: Cache pip
+        uses: actions/cache@v4
+        env:
+          # Increase this value to reset cache
+          CACHE_NUMBER: 0
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-doc_build-pip-${{ env.CACHE_NUMBER }}
+          restore-keys: |
+            ${{ runner.os }}-doc_build-pip-${{ env.CACHE_NUMBER }}
+
+
       - name: Install package in Ubuntu
         if: ${{ inputs.install_package_ubuntu }}
         run : |
@@ -94,7 +117,7 @@ jobs:
       - uses: actions/setup-python@v5
         name: Install Python
         with:
-          python-version: '3.11'
+          python-version: '3.12'
 
       - name: Install library
         if: ${{ inputs.install_dependencies_only != true }}
@@ -106,6 +129,47 @@ jobs:
         run: |
           pip install -r requirements.txt
 
+      - name: Cache doc build
+        if: ${{ inputs.CACHE_GALLERY_EXAMPLES }}
+        uses: actions/cache@v4
+        env:
+          # Increase this value to reset cache
+          CACHE_NUMBER: 0
+        with:
+          path: ${{ inputs.CACHE_GALLERY_EXAMPLES }}
+          key: ${{ runner.os }}-doc_build-gallery-${{ env.CACHE_NUMBER }}
+          restore-keys: |
+            ${{ inputs.CACHE_GALLERY_EXAMPLES }}
+
+      - name: Display content of the cached doc build folder
+        if: ${{ inputs.CACHE_GALLERY_EXAMPLES }}
+        run: |
+            if [ -d "${{ inputs.CACHE_GALLERY_EXAMPLES }}" ]; then
+              echo "Content of the cached doc build folder:"
+              ls -l ${{ inputs.CACHE_GALLERY_EXAMPLES }}
+            else
+              echo "Doc build folder does not exist (yet?): ${{ inputs.CACHE_GALLERY_EXAMPLES }}"
+            fi
+
+      - name: Get pooch cache paths
+        if: ${{ inputs.CACHE_POOCH != '' }}
+        run: |
+          # create list of pooch cache paths as a string (space separated)
+          python -c "import pooch; print(' '.join([str(pooch.os_cache('${{ inputs.CACHE_POOCH }}')), str(pooch.os_cache('pooch'))]))" > pooch_cache_paths.txt
+          echo "POOCH_CACHE_PATHS=$(cat pooch_cache_paths.txt)" >> $GITHUB_ENV
+
+      - name: Cache pooch
+        if: ${{ inputs.CACHE_POOCH != ''}}
+        uses: actions/cache@v4
+        env:
+          # Increase this value to reset cache
+          CACHE_NUMBER: 0
+        with:
+          path: ${{ env.POOCH_CACHE_PATHS }}
+          key: ${{ runner.os }}-doc_build-pooch-${{ env.CACHE_NUMBER }}
+          restore-keys: |
+            ${{ runner.os }}-doc_build-pooch-${{ env.CACHE_NUMBER }}
+
       - name: Run doctest
         if: ${{ inputs.doctest }}
         run: |
@@ -121,7 +185,7 @@ jobs:
           FORCE_COLOR: 1
         run: |
           cd ${{ inputs.doc_path }}
-          make SPHINXOPTS=${{ inputs.SPHINXOPTS }} html
+          make SPHINXOPTS="${{ inputs.SPHINXOPTS }}" html
 
       - name: Build documentation (with xvfb)
         if: ${{ inputs.use_xvfb }}
@@ -129,7 +193,7 @@ jobs:
           FORCE_COLOR: 1
         run: |
           cd ${{ inputs.doc_path }}
-          xvfb-run make SPHINXOPTS=${{ inputs.SPHINXOPTS }} html
+          xvfb-run make SPHINXOPTS="${{ inputs.SPHINXOPTS }}" html
 
       - uses: actions/upload-artifact@v4
         with:

From 8b6cd3bebb8c8dd936c8f4e9d4f3a71d59574b60 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Fri, 25 Jul 2025 09:43:40 +0100
Subject: [PATCH 24/25] Add caching in `package_and_test.yml` workflow

---
 .github/workflows/package_and_test.yml | 54 ++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/.github/workflows/package_and_test.yml b/.github/workflows/package_and_test.yml
index 382e095..cb5ecc8 100644
--- a/.github/workflows/package_and_test.yml
+++ b/.github/workflows/package_and_test.yml
@@ -42,6 +42,10 @@ on:
         description: 'The organisation hosting the upstream code'
         default: 'hyperspy'
         type: string  
+      CACHE_POOCH:
+        description: 'The list of pooch cache to cache - see pooch.os_cache() for more information.'
+        default: ''
+        type: string
 
 env:
   PYTHON_VERSION: '3.11'
@@ -163,6 +167,56 @@ jobs:
         run: |
           pip install ${{ inputs.ADDITIONAL_TEST_DEPENDENCIES }}
 
+      - name: Cache pip
+        uses: actions/cache@v4
+        env:
+          # Increase this value to reset cache
+          CACHE_NUMBER: 0
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-package&test-pip-${{ env.CACHE_NUMBER }}
+          restore-keys: |
+            ${{ runner.os }}-package&test-pip-${{ env.CACHE_NUMBER }}
+
+      - name: Get pooch cache paths
+        if: ${{ inputs.CACHE_POOCH != '' }}
+        run: |
+          # create list of pooch cache paths as a string (space separated)
+          python -c "import pooch; print(' '.join([str(pooch.os_cache(s)) for s in ${{ inputs.CACHE_POOCH }}.split(' ')])))" > pooch_cache_paths.txt
+          echo "POOCH_CACHE_PATHS=$(cat pooch_cache_paths.txt)" >> $GITHUB_ENV
+
+      - name: Cache pooch data
+        if: ${{ inputs.CACHE_POOCH != '' }}
+        uses: actions/cache@v4
+        env:
+          # Increase this value to reset cache
+          CACHE_NUMBER: 0
+        with:
+          path: ${{ env.POOCH_CACHE_PATHS }}
+          key: ${{ runner.os }}-package&test-pooch-${{ env.CACHE_NUMBER }}
+          restore-keys: |
+            ${{ runner.os }}-package&test-pooch-${{ env.CACHE_NUMBER }}
+
+      - name: Get rosettasciio tests data location
+        # special case rosettasciio to cache the test data
+        if: ${{ inputs.module_name == 'rsciio' }}
+        run: |
+          python -c "from rsciio.tests import registry; print(f'ROSETTASCIIO_TEST_DATA={str(registry.TESTS_PATH / \"data\") }')" >> $GITHUB_ENV
+
+      - name: Cache rosettasciio data
+        # special case rosettasciio to cache the test data
+        if: ${{ inputs.module_name == 'rsciio' }}
+        uses: actions/cache@v4
+        env:
+          # Increase this value to reset cache
+          CACHE_NUMBER: 0
+        with:
+          path: ${{ env.ROSETTASCIIO_TEST_DATA }}
+          key:
+            ${{ runner.os }}-package&test-test_data-${{ env.CACHE_NUMBER }}
+          restore-keys: |
+            ${{ runner.os }}-package&test-test_data-${{ env.CACHE_NUMBER }}
+
       - name: Pip list
         run: |
           pip list

From 43db1399b9aad35d09de75a35f80aab02ad5f0c6 Mon Sep 17 00:00:00 2001
From: Eric Prestat <eric.prestat@gmail.com>
Date: Sun, 27 Jul 2025 08:44:00 +0100
Subject: [PATCH 25/25] Add step to trigger readthedocs build

---
 .github/workflows/doc.yml | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml
index 8eadb26..f09108d 100644
--- a/.github/workflows/doc.yml
+++ b/.github/workflows/doc.yml
@@ -53,7 +53,13 @@ on:
                       The default pooch cache (pooch.os_cache("pooch")) will also be cached.
                       If empty (default), no caching is performed.'
         default: ''
-        type: string  
+        type: string
+      READTHEDOCS_TOKEN:
+        description: 'NOT TESTED. User token used to trigger the ReadTheDocs build. 
+                      If empty (default), no ReadTheDocs build is triggered.
+                      The token can be created in the ReadTheDocs user settings.'
+        default: ''
+        type: string
 
 
 jobs:
@@ -207,3 +213,14 @@ jobs:
         run: |
           cd ${{ inputs.doc_path }}
           make linkcheck
+
+      - name: Trigger ReadTheDocs build
+        # NOT TESTED
+        if: ${{ inputs.READTHEDOCS_TOKEN }}
+        env:
+          RTD_PROJECT: ${{ env.REPOSITORY_NAME }}
+          RTD_TOKEN: ${{ inputs.READTHEDOCS_TOKEN }}
+          RTD_VERSION: ${{ github.event_name == 'push' && 'latest' || (github.event_name == 'pull_request' && format('pull/{0}', github.event.pull_request.number)) || (startsWith(github.ref, 'refs/tags/') && format('v{0}', github.ref_name)) }}
+        run: |
+          curl -X POST "https://readthedocs.org/api/v3/projects/${RTD_PROJECT}/versions/${RTD_VERSION}/builds/" -H "Authorization: Token ${RTD_TOKEN}"
+