diff --git a/.coveragerc b/.coveragerc index 6e20457..97b913b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,6 +4,8 @@ omit = drms/*setup_package* drms/extern/* drms/version* + drms/_dev/* + */drms/_dev/* */drms/conftest.py */drms/*setup_package* */drms/extern/* diff --git a/.cruft.json b/.cruft.json index 065cb23..ac67e32 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/sunpy/package-template", - "commit": "29db3569a215e43fa8c5114d3dbcb98b6a9caac0", + "commit": "cc15a3492324a0415ea444c5f52b1913323ad3ba", "checkout": null, "context": { "cookiecutter": { @@ -17,12 +17,13 @@ "changelog_url": "https://docs.sunpy.org/projects/drms/en/stable/whatsnew/changelog.html", "issue_tracker_url": "https://github.com/sunpy/drms/issues", "license": "BSD 2-Clause", - "minimum_python_version": "3.10", + "minimum_python_version": "3.12", "use_compiled_extensions": "n", "enable_dynamic_dev_versions": "y", "include_example_code": "n", "include_cruft_update_github_workflow": "y", "use_extended_ruff_linting": "y", + "matrix_room_id": "", "_sphinx_theme": "sunpy", "_parent_project": "", "_install_requires": "", @@ -32,7 +33,7 @@ ".github/workflows/sub_package_update.yml" ], "_template": "https://github.com/sunpy/package-template", - "_commit": "29db3569a215e43fa8c5114d3dbcb98b6a9caac0" + "_commit": "cc15a3492324a0415ea444c5f52b1913323ad3ba" } }, "directory": null diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..e8eb01e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + cooldown: + default-days: 7 + groups: + actions: + patterns: + - "*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11e0bed..6f1c1a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,32 +21,36 @@ on: # │ │ ┌───────── day of the month (1 - 31) # │ │ │ ┌───────── month (1 - 12 or JAN-DEC) # │ │ │ │ ┌───────── day of the week (0 - 6 or SUN-SAT) - - cron: '0 7 * * *' # Every day at 07:00 UTC + - cron: '0 7 * * 3' # Every Wed at 07:00 UTC concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} + jobs: core: - uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v2 # zizmor: ignore[unpinned-uses] with: submodules: false coverage: codecov toxdeps: "tox-pypi-filter" posargs: -n auto envs: | - - linux: py312 + - linux: py313 secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} sdist_verify: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 # zizmor: ignore[unpinned-uses] + with: + persist-credentials: false + - uses: actions/setup-python@v6 # zizmor: ignore[unpinned-uses] with: - python-version: '3.12' + python-version: '3.13' - run: python -m pip install -U --user build - run: python -m build . --sdist - run: python -m pip install -U --user twine @@ -54,24 +58,26 @@ jobs: test: needs: [core, sdist_verify] - uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v2 # zizmor: ignore[unpinned-uses] with: submodules: false coverage: codecov toxdeps: "tox-pypi-filter" posargs: -n auto envs: | - - windows: py10 - - macos: py311 - - linux: py310-oldestdeps + - linux: py314 + - windows: py312 + - macos: py312 + - linux: py312-oldestdeps + - linux: py314-devdeps secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} docs: needs: [core] - uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v2 # zizmor: ignore[unpinned-uses] with: - default_python: '3.12' + default_python: '3.13' submodules: false pytest: false toxdeps: "tox-pypi-filter" @@ -82,7 +88,7 @@ jobs: - linux: build_docs online: - uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 + uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v2 # zizmor: ignore[unpinned-uses] with: default_python: '3.12' submodules: false @@ -95,7 +101,7 @@ jobs: secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - publish: + build_dists: # Build wheels on PRs only when labelled. Releases will only be published if tagged ^v.* # see https://github-actions-workflows.openastronomy.org/en/latest/publish.html#upload-to-pypi if: | @@ -105,11 +111,33 @@ jobs: contains(github.event.pull_request.labels.*.name, 'Run publish') ) needs: [test, docs] - uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v1 + uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v2 # zizmor: ignore[unpinned-uses] with: - python-version: '3.12' + python-version: '3.13' test_extras: 'tests' test_command: 'pytest -p no:warnings --doctest-rst --pyargs drms' submodules: false - secrets: - pypi_token: ${{ secrets.pypi_token }} + save_artifacts: true + upload_to_pypi: false + + publish: + if: startsWith(github.ref, 'refs/tags/v') + name: Upload to PyPI + runs-on: ubuntu-latest + needs: [build_dists] + permissions: + id-token: write + environment: + name: pypi + steps: + - name: Download artifacts + uses: actions/download-artifact@v8 # zizmor: ignore[unpinned-uses] + with: + merge-multiple: true + pattern: dist-* + path: dist + + - run: ls -lha dist/ + + - name: Run upload + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 diff --git a/.github/workflows/sub_package_update.yml b/.github/workflows/sub_package_update.yml index 523cca3..bfb213a 100644 --- a/.github/workflows/sub_package_update.yml +++ b/.github/workflows/sub_package_update.yml @@ -1,9 +1,6 @@ # This template is taken from the cruft example code, for further information please see: # https://cruft.github.io/cruft/#automating-updates-with-github-actions name: Automatic Update from package template -permissions: - contents: write - pull-requests: write on: # Allow manual runs through the web UI @@ -16,17 +13,22 @@ on: # │ │ │ │ ┌───────── day of the week (0 - 6 or SUN-SAT) - cron: '0 7 * * 1' # Every Monday at 7am UTC +permissions: {} + jobs: update: runs-on: ubuntu-latest - strategy: - fail-fast: true + permissions: + contents: write + pull-requests: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 # zizmor: ignore[unpinned-uses] + with: + persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 # zizmor: ignore[unpinned-uses] with: - python-version: "3.11" + python-version: "3.14" - name: Install Cruft run: python -m pip install git+https://github.com/Cadair/cruft@patch-p1 @@ -50,8 +52,8 @@ jobs: id: cruft_update if: steps.check.outputs.has_changes == '1' run: | - git config --global user.email "${{ github.actor }}@users.noreply.github.com" - git config --global user.name "${{ github.actor }}" + git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" + git config --global user.name "${GITHUB_ACTOR}" cruft_output=$(cruft update --skip-apply-ask --refresh-private-variables) echo $cruft_output @@ -77,7 +79,7 @@ jobs: - name: Create pull request if: steps.cruft_json.outputs.has_changes == '1' - uses: peter-evans/create-pull-request@v7 + uses: peter-evans/create-pull-request@v8 # zizmor: ignore[unpinned-uses] with: token: ${{ secrets.GITHUB_TOKEN }} add-paths: "." @@ -93,3 +95,70 @@ jobs: If this pull request has been opened as a draft there are conflicts which need fixing. **To run the CI on this pull request you will need to close it and reopen it.** + + report-fail: + if: failure() + needs: [update] + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Open an issue if workflow fails + uses: actions/github-script@v9 # zizmor: ignore[unpinned-uses] + with: + github-token: ${{ github.token }} + # This script is adapted from https://github.com/scientific-python/issue-from-pytest-log-action + # Under MIT license (c) Scientific Python Developers + script: | + const fs = require('fs'); + + // Edit these if needed for your repo + const variables = { + owner: context.repo.owner, + name: context.repo.repo, + label: "Infrastructure", + creator: "app/github-actions", + title: "SunPy Package Template auto-update failed." + }; + + const logs = 'The package update workflow failed.' + const workflow_url = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; + const issue_body = `[Workflow Run URL](${workflow_url})\n${logs}`; + + const query_string = `repo:${variables.owner}/${variables.name} author:${variables.creator} label:${variables.label} is:open in:title ${variables.title}`; + + // Run GraphQL query against GitHub API to find the most recent open issue used for reporting failures + const query = `query { + search(query: "${query_string}", type:ISSUE, first: 1) { + edges { + node { + ... on Issue { + body + id + number + } + } + } + } + }`; + + const result = await github.graphql(query); + + // If no issue is open, create a new issue, + // else update the body of the existing issue. + if (result.search.edges.length === 0) { + await github.rest.issues.create({ + owner: variables.owner, + repo: variables.name, + body: issue_body, + title: variables.title, + labels: [variables.label, "pre-commit.ci autofix"], + }); + } else { + await github.rest.issues.update({ + owner: variables.owner, + repo: variables.name, + issue_number: result.search.edges[0].node.number, + body: issue_body + }); + } diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4d918f8..f30f03e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,36 +1,59 @@ -exclude: ".*(.fits|.fts|.fit|.header|.txt|tca.*|extern.*|drms/extern)$|^CITATION.rst$" - repos: + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.24.1 + hooks: + - id: zizmor + # This should be before any formatting hooks like isort - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.9.4" + rev: "v0.15.12" hooks: - id: ruff - args: ['--fix', '--unsafe-fixes'] + args: ["--fix"] + types: [python] + # Define here once and then reference using YAML anchor + exclude: &exclude_dirs ^drms/(data|extern)/ - id: ruff-format + types: [python] + exclude: *exclude_dirs - repo: https://github.com/PyCQA/isort - rev: 6.0.0 - hooks: - - id: isort - - repo: https://github.com/PyCQA/isort - rev: 6.0.0 + rev: 8.0.1 hooks: - id: isort + types: [python] + exclude: *exclude_dirs - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-ast + types: [python] + exclude: *exclude_dirs - id: check-case-conflict + types: [python] + exclude: *exclude_dirs - id: trailing-whitespace + types_or: [python, rst] - id: check-yaml + types: [yaml] + exclude: *exclude_dirs + - id: check-toml + types: [toml] + exclude: *exclude_dirs - id: debug-statements + types: [python] + exclude: *exclude_dirs - id: check-added-large-files args: ["--enforce-all", "--maxkb=1054"] - id: end-of-file-fixer + types_or: [python, rst] - id: mixed-line-ending + types_or: [python, rst] - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 + rev: v2.4.2 hooks: - id: codespell + args: [ "--write-changes" ] + types_or: [python, rst] + exclude: *exclude_dirs ci: autofix_prs: false autoupdate_schedule: "quarterly" diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index 5678670..0000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: 2 -build: - os: ubuntu-20.04 - tools: - python: "3.11" - apt_packages: - - graphviz - -sphinx: - builder: html - configuration: docs/conf.py - fail_on_warning: false - -python: - install: - - method: pip - extra_requirements: - - all - - docs - path: . diff --git a/.ruff.toml b/.ruff.toml index d33105a..682afa9 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -34,8 +34,6 @@ select = [ extend-ignore = [ # pycodestyle (E, W) "E501", # ignore line length will use a formatter instead - # pyupgrade (UP) - "UP038", # Use | in isinstance - not compatible with models and is slower # pytest (PT) "PT001", # Always use pytest.fixture() "PT023", # Always use () on pytest decorators diff --git a/158.breaking.rst b/158.breaking.rst new file mode 100644 index 0000000..492cadc --- /dev/null +++ b/158.breaking.rst @@ -0,0 +1,3 @@ +Increased minimum version of Python to 3.12. +Increased minimum version of NumPy to 1.26.0. +Increased minimum version of pandas to 2.2.0. diff --git a/MANIFEST.in b/MANIFEST.in index ab1f5d3..6492c4a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,11 +1,20 @@ # Exclude specific files # All files which are tracked by git and not explicitly excluded here are included by setuptools_scm +exclude .codecov.yaml +exclude .coveragerc +exclude .cruft.json +exclude .gitignore +exclude .isort.cfg +exclude .pre-commit-config.yaml +exclude .readthedocs.yaml +exclude .rtd-environment.yml +exclude .ruff.toml + # Prune folders +prune .github prune build -prune docs/_build -prune docs/api +prune changelog global-exclude *.pyc *.o - # This subpackage is only used in development checkouts # and should not be included in built tarballs prune drms/_dev diff --git a/README.rst b/README.rst index 4532d91..3d1529a 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,7 @@ -==== -drms -==== +``drms`` +======== -Access HMI, AIA and MDI data from the Standford JSOC DRMS. +Access HMI, AIA and MDI data from the Standford JSOC DRMS `Docs `__ | `Tutorial `__ | @@ -28,14 +27,13 @@ For more information or to ask questions about ``drms``, check out: - `drms Documentation `__ - `SunPy Chat `__ -License -------- +Usage of Generative AI +---------------------- -This project is Copyright (c) The SunPy Community and licensed under -the terms of the BSD 2-Clause license. This package is based upon -the `Openastronomy packaging guide `_ -which is licensed under the BSD 3-clause licence. See the licenses folder for -more information. +We expect authentic engagement in our community. +**Do not post the output from Large Language Models or similar generative AI as code, issues or comments on GitHub or any other platform.** +If you use generative AI tools as an aid in developing code or documentation changes, ensure that you fully understand the proposed changes and can explain why they are the correct approach and an improvement to the current state. +For more information see our documentation on fair and appropriate `AI usage `__. Contributing ------------ @@ -43,31 +41,18 @@ Contributing We love contributions! drms is open source, built on open source, and we'd love to have you hang out in our community. -**Imposter syndrome disclaimer**: We want your help. No, really. +If you would like to get involved, check out the `Developers Guide`_ section of the SunPy docs. +Stop by our chat room `#sunpy:openastronomy.org`_ if you have any questions. +Help is always welcome so let us know what you like to work on, or check out the `issues page`_ for the list of known outstanding items. -There may be a little voice inside your head that is telling you that you're not -ready to be an open source contributor; that your skills aren't nearly good -enough to contribute. What could you possibly offer a project like this one? +For more information on contributing to SunPy, please read our `Newcomers' guide`_. -We assure you - the little voice in your head is wrong. If you can write code at -all, you can contribute code to open source. Contributing to open source -projects is a fantastic way to advance one's coding skills. Writing perfect code -isn't the measure of a good developer (that would disqualify all of us!); it's -trying to create something, making mistakes, and learning from those -mistakes. That's how we all improve, and we are happy to help others learn. +.. _Developers Guide: https://docs.sunpy.org/en/latest/dev_guide/index.html +.. _`#sunpy:openastronomy.org`: https://app.element.io/#/room/#sunpy:openastronomy.org +.. _issues page: https://github.com/sunpy/drms/issues +.. _Newcomers' guide: https://docs.sunpy.org/en/latest/dev_guide/contents/newcomers.html -Being an open source contributor doesn't just mean writing code, either. You can -help out by writing documentation, tests, or even giving feedback about the -project (and yes - that includes giving feedback about the contribution -process). Some of these contributions may be the most valuable to the project as -a whole, because you're coming to the project with fresh eyes, so you can see -the errors and assumptions that seasoned contributors have glossed over. - -Note: This disclaimer was originally written by -`Adrienne Lowe `_ for a -`PyCon talk `_, and was adapted by -drms based on its use in the README file for the -`MetPy project `_. +When you are interacting with the SunPy community you are asked at to follow our `code of conduct `__. Citation -------- @@ -88,11 +73,6 @@ If you use ``drms`` in your work, please cite our `paper `__. - Acknowledgements ---------------- diff --git a/changelog/156.doc.rst b/changelog/156.doc.rst new file mode 100644 index 0000000..779872b --- /dev/null +++ b/changelog/156.doc.rst @@ -0,0 +1 @@ +Added export status codes to docstring of `drms.ExportResult.status`. diff --git a/changelog/169.bugfix.rst b/changelog/169.bugfix.rst new file mode 100644 index 0000000..745f98e --- /dev/null +++ b/changelog/169.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug in parsing of hexadecimal columns. diff --git a/docs/conf.py b/docs/conf.py index 3a011b4..8bb126f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,11 +34,15 @@ # -- General configuration --------------------------------------------------- +# Wrap large function/method signatures +maximum_signature_line_length = 80 + # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named "sphinx.ext.*") or your custom # ones. extensions = [ - "hoverxref.extension", + # TODO: Re-enable this, it was failing the doc build + # "hoverxref.extension", "sphinx_copybutton", "sphinx_gallery.gen_gallery", "sphinx.ext.autodoc", @@ -65,8 +69,7 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -source_suffix = ".rst" +source_suffix = {".rst": "restructuredtext"} # The master toctree document. master_doc = "index" diff --git a/docs/tutorial.rst b/docs/tutorial.rst index c4bed38..201426e 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -91,7 +91,7 @@ JSOC time strings can be converted to a naive `~datetime.datetime` representatio 1 2016-04-01 06:00:00 2 2016-04-01 12:00:00 3 2016-04-01 18:00:00 - Name: T_REC, dtype: datetime64[ns] + Name: T_REC, dtype: datetime64[us] For most of the HMI and MDI data sets, the `TAI `__ time standard is used which, in contrast to `UTC `__, does not make use of any leap seconds. The TAI standard is currently not supported by the Python standard libraries. @@ -210,7 +210,7 @@ Note that :meth:`drms.client.Client.export` performs an ``url_quick`` / ``as-is` 1 /SUM41/D803708361/S00008/Dopplergram.fits 2 /SUM71/D803720859/S00008/Dopplergram.fits 3 /SUM70/D803730119/S00008/Dopplergram.fits - Name: filename, dtype: object + Name: filename, dtype: str Download URLs can now be generated using the :attr:`drms.client.ExportRequest.urls` attribute: @@ -225,7 +225,7 @@ The following, for example, only downloads the first file of the request: .. code-block:: python - >>> export_request.download(out_dir, index=0) # doctest: +REMOTE_DATA + >>> res = export_request.download(out_dir, index=0) # doctest: +REMOTE_DATA Being a direct ``as-is`` export, there are no keyword data written to any FITS headers. If you need keyword data added to the headers, you have to use the ``fits`` export protocol instead, which is described below. diff --git a/drms/__init__.py b/drms/__init__.py index 92bb12e..e68050b 100644 --- a/drms/__init__.py +++ b/drms/__init__.py @@ -29,7 +29,7 @@ def _get_bibtex(): - import textwrap + import textwrap # noqa: PLC0415 # Set the bibtex entry to the article referenced in CITATION.rst citation_file = Path(__file__).parent / "CITATION.rst" diff --git a/drms/client.py b/drms/client.py index c9bc0b4..787734c 100644 --- a/drms/client.py +++ b/drms/client.py @@ -11,6 +11,7 @@ import numpy as np import pandas as pd +from pandas.api.types import is_object_dtype, is_string_dtype from drms import logger from drms.utils import create_request_with_header @@ -239,6 +240,20 @@ def id(self): def status(self): """ (int) Export request status. + + Available status codes: + + - 0 = OK immediate data available or queue managed data is complete. + - 1 = Request received and action is pending, i.e. in processing. + - 2 = Queued for processing. + - 3 = Request too large for automatic requests. + - 4 = Request not formed correctly, bad series, etc. + - 5 = Request old, results requested after data timed out. + - -1 = The backend process was terminated (typically when the user cancels the export request). + + Reference + --------- + http://jsoc.stanford.edu/jsocwiki/AjaxJsocConnect#line-27-2 """ return self._status @@ -457,11 +472,12 @@ def wait(self, *, timeout=None, sleep=5, retries_notfound=5): if t_start + timeout + wait_secs - time.time() < 0: return False - logger.info(f"Waiting for {int(round(wait_secs))} seconds...") + logger.info(f"Waiting for {round(wait_secs)} seconds...") time.sleep(wait_secs) if self.has_finished(): self._raise_on_error() + logger.info(f"Export request finished. [id={idstr}, status={self._status}]") return True if self._status == self._status_code_notfound: # Raise exception, if no retries are left. @@ -600,10 +616,9 @@ def __init__(self, server="jsoc", *, email=None): def __repr__(self): return f"" - def _convert_numeric_keywords(self, ds, kdf, *, skip_conversion=None): - si = self.info(ds) - int_keys = list(si.keywords[si.keywords.is_integer].index) - num_keys = list(si.keywords[si.keywords.is_numeric].index) + def _convert_numeric_keywords(self, keywords, kdf, *, skip_conversion=None): + int_keys = list(keywords[keywords.is_integer].index) + num_keys = list(keywords[keywords.is_numeric].index) num_keys += ["*recnum*", "*sunum*", "*size*"] if skip_conversion is None: skip_conversion = [] @@ -616,11 +631,12 @@ def _convert_numeric_keywords(self, ds, kdf, *, skip_conversion=None): # we need a special treatment for integer strings that start # with '0x', like QUALITY. The following to_numeric call is # still necessary as the results are still Python objects. - if k in int_keys and kdf[k].dtype is np.dtype(object): - idx = kdf[k].str.startswith("0x") + if k in int_keys and (is_object_dtype(kdf[k]) or is_string_dtype(kdf[k])): + values = kdf[k].astype(str) + idx = values.str.startswith(("0x", "0X")) if idx.any(): - k_idx = kdf.columns.get_loc(k) - kdf.loc[idx, kdf.columns[k_idx]] = kdf.loc[idx, kdf.columns[k_idx]].apply(int, base=16) + kdf[k] = kdf[k].astype(object) + kdf.loc[idx, k] = values[idx].apply(int, base=16) if k in num_keys: kdf[k] = _pd_to_numeric_coerce(kdf[k]) @@ -1015,7 +1031,7 @@ def query( else: res_key = pd.DataFrame() if convert_numeric: - self._convert_numeric_keywords(ds, res_key, skip_conversion=skip_conversion) + self._convert_numeric_keywords(self.info(ds).keywords, res_key, skip_conversion=skip_conversion) res.append(res_key) if seg is not None: diff --git a/drms/json.py b/drms/json.py index fe9d815..df004fb 100644 --- a/drms/json.py +++ b/drms/json.py @@ -225,7 +225,7 @@ def rs_list(self, ds, *, key=None, seg=None, link=None, recinfo=False, n=None, u if recinfo: d["R"] = "1" if n is not None: - d["n"] = f"{int(int(n))}" + d["n"] = f"{int(n)}" if uid is not None: d["userhandle"] = uid query = f"?{urlencode(d)}" diff --git a/drms/main.py b/drms/main.py index 162321c..6dc70ab 100644 --- a/drms/main.py +++ b/drms/main.py @@ -3,7 +3,7 @@ def main(): - import drms + import drms # noqa: PLC0415 args = parse_args(sys.argv[1:]) client = drms.Client(server=args.server, email=args.email) @@ -11,7 +11,7 @@ def main(): def parse_args(args): - import drms + import drms # noqa: PLC0415 parser = argparse.ArgumentParser(description="drms, access HMI, AIA and MDI data with python") parser.add_argument( diff --git a/drms/tests/conftest.py b/drms/tests/conftest.py index d68cbfd..0f78fc8 100644 --- a/drms/tests/conftest.py +++ b/drms/tests/conftest.py @@ -4,6 +4,7 @@ import pytest +import drms from drms.utils import create_request_with_header # Test URLs, used to check if a online site is reachable @@ -69,8 +70,6 @@ def jsoc_client(): """ Client fixture for JSOC online tests, does not use email. """ - import drms - return drms.Client("jsoc") @@ -79,8 +78,6 @@ def jsoc_client_export(email): """ Client fixture for JSOC online tests, uses email if specified. """ - import drms - return drms.Client("jsoc", email=email) @@ -89,6 +86,4 @@ def kis_client(): """ Client fixture for KIS online tests. """ - import drms - return drms.Client("kis") diff --git a/drms/tests/test_jsoc_export.py b/drms/tests/test_jsoc_export.py index e011a80..4ef7af8 100644 --- a/drms/tests/test_jsoc_export.py +++ b/drms/tests/test_jsoc_export.py @@ -149,5 +149,5 @@ def test_export_invalid_process(jsoc_client_export): @pytest.mark.jsoc() @pytest.mark.remote_data() def test_export_email(jsoc_client): - with pytest.raises(ValueError, match="The email argument is required, when no default email address was set."): + with pytest.raises(ValueError, match=r"The email argument is required, when no default email address was set."): jsoc_client.export("hmi.v_45s[2016.04.01_TAI/1d@6h]{Dopplergram}") diff --git a/drms/tests/test_jsoc_query.py b/drms/tests/test_jsoc_query.py index 659cb25..e1463f0 100644 --- a/drms/tests/test_jsoc_query.py +++ b/drms/tests/test_jsoc_query.py @@ -1,3 +1,4 @@ +import pandas as pd import pytest import drms @@ -100,9 +101,20 @@ def test_query_invalid_series(jsoc_client): [ "hmi.v_45s[2014.01.01_00:00:35_TAI-2014.01.01_01:00:35_TAI]", "hmi.M_720s[2011.04.14_00:30:00_TAI/6h@2h]", + "aia.lev1_euv_12s[2014-01-01T00:00:01Z/365d@1d][335]", ], ) def test_query_hexadecimal_strings(query): # Exercise the part of client.py that deals with hexadecimal strings c = drms.Client() - c.query(query, key="**ALL**") + result = c.query(query, key=["T_REC", "QUALITY", "CRPIX1", "CRVAL1", "BUNIT"]) + assert pd.api.types.is_integer_dtype(result["QUALITY"]) + + +def test_query_quality_hex_decimal_conversion(): + c = drms.Client() + keywords = pd.DataFrame({"is_integer": [True], "is_numeric": [True]}, index=["QUALITY"]) + df = pd.DataFrame({"QUALITY": pd.Series(["0x00000000", "0x0000000A", "0X000000FF"], dtype="string")}) + c._convert_numeric_keywords(keywords, df) + assert df["QUALITY"].tolist() == [0, 10, 255] + assert pd.api.types.is_integer_dtype(df["QUALITY"]) diff --git a/drms/tests/test_to_datetime.py b/drms/tests/test_to_datetime.py index 5deea8f..7863f8a 100644 --- a/drms/tests/test_to_datetime.py +++ b/drms/tests/test_to_datetime.py @@ -82,7 +82,6 @@ def test_time_series(time_series, expected): ("2010.05.01_TAI", False), ("2010.05.01_00:00_TAI", False), ("", True), - ("1600", True), ("foo", True), ("2013.12.21_23:32:34_TAI", False), ] diff --git a/examples/cutout_export_request.py b/examples/cutout_export_request.py index e41f86d..6514beb 100644 --- a/examples/cutout_export_request.py +++ b/examples/cutout_export_request.py @@ -69,7 +69,7 @@ # Print request URL. print(f"\nRequest URL: {result.request_url}") -print(f"{int(len(result.urls))} file(s) available for download.\n") +print(f"{len(result.urls)} file(s) available for download.\n") # Download selected files. result.wait() diff --git a/examples/export_as_is.py b/examples/export_as_is.py index 44cb59a..622e97b 100644 --- a/examples/export_as_is.py +++ b/examples/export_as_is.py @@ -40,7 +40,7 @@ # Submit export request, defaults to method='url_quick' and protocol='as-is' print("Submitting export request...") result = client.export(qstr, email=email) -print(f"{int(len(result.urls))} file(s) available for download.\n") +print(f"{len(result.urls)} file(s) available for download.\n") # Download selected files. result.download(out_dir) diff --git a/examples/export_fits.py b/examples/export_fits.py index 7abc821..8d71fb5 100644 --- a/examples/export_fits.py +++ b/examples/export_fits.py @@ -47,7 +47,7 @@ # Print request URL. print(f"\nRequest URL: {result.request_url}") -print(f"{int(len(result.urls))} file(s) available for download.\n") +print(f"{len(result.urls)} file(s) available for download.\n") # Download selected files. result.download(out_dir) diff --git a/examples/export_jpg.py b/examples/export_jpg.py index 79e6310..8254065 100644 --- a/examples/export_jpg.py +++ b/examples/export_jpg.py @@ -53,7 +53,7 @@ # Print request URL. print(f"\nRequest URL: {result.request_url}") -print(f"{int(len(result.urls))} file(s) available for download.\n") +print(f"{len(result.urls)} file(s) available for download.\n") # Download selected files. result.download(out_dir) diff --git a/examples/plot_aia_lightcurve.py b/examples/plot_aia_lightcurve.py index dc23cf6..9505085 100644 --- a/examples/plot_aia_lightcurve.py +++ b/examples/plot_aia_lightcurve.py @@ -40,9 +40,10 @@ print("Querying series info...") series_info = client.info("aia.lev1_euv_12s") series_info_lev1 = client.info("aia.lev1") +linkinfo = series_info.keywords.linkinfo +missing_linkinfo = linkinfo.isna() for key in keys: - linkinfo = series_info.keywords.loc[key].linkinfo - if linkinfo is not None and linkinfo.startswith("lev1->"): + if not missing_linkinfo.loc[key] and linkinfo.loc[key].startswith("lev1->"): note_str = series_info_lev1.keywords.loc[key].note else: note_str = series_info.keywords.loc[key].note diff --git a/examples/plot_hmi_lightcurve.py b/examples/plot_hmi_lightcurve.py index 7cfb199..34fb609 100644 --- a/examples/plot_hmi_lightcurve.py +++ b/examples/plot_hmi_lightcurve.py @@ -23,7 +23,7 @@ # Send request to the DRMS server print(f"Querying keyword data...\n -> {qstr}") result = client.query(qstr, key=["T_REC", "DATAMEAN", "DATARMS"]) -print(f" -> {int(len(result))} lines retrieved.") +print(f" -> {len(result)} lines retrieved.") ############################################################################### # Now to plot the image. diff --git a/examples/skip_export_from_id.py b/examples/skip_export_from_id.py index 8385412..6f4b778 100644 --- a/examples/skip_export_from_id.py +++ b/examples/skip_export_from_id.py @@ -28,7 +28,7 @@ # Print request URL and number of available files. print(f"\nRequest URL: {result.request_url}") -print(f"{int(len(result.urls))} file(s) available for download.\n") +print(f"{len(result.urls)} file(s) available for download.\n") # Create download directory if it does not exist yet. out_dir = Path("downloads") / request_id diff --git a/pyproject.toml b/pyproject.toml index 1a1f82c..8124df6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,27 +9,26 @@ build-backend = "setuptools.build_meta" [project] name = "drms" description = "Access HMI, AIA and MDI data from the Standford JSOC DRMS" -requires-python = ">=3.10" +requires-python = ">=3.12" readme = { file = "README.rst", content-type = "text/x-rst" } -license = { file = "licenses/LICENSE.rst" } +license-files = ["licenses/LICENSE.rst"] authors = [ { name = "The SunPy Community", email = "sunpy@googlegroups.com" }, ] dependencies = [ - "numpy>=1.23.5", - "pandas>=1.5.1", - "packaging>=23.0" + "numpy>=1.26.0", + "pandas>=2.2.0", ] dynamic = [ "version" ] [project.optional-dependencies] tests = [ + "astropy", # Doctests "pytest", "pytest-astropy", "pytest-doctestplus", "pytest-cov", "pytest-xdist", - "astropy", ] docs = [ "sphinx", diff --git a/pytest.ini b/pytest.ini index e2bf1c7..dac2b3b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -39,6 +39,8 @@ filterwarnings = # A list of warnings to ignore follows. If you add to this list, you MUST # add a comment or ideally a link to an issue that explains why the warning # is being ignored + # Lmao Pandas + ignore:(?s).*Pyarrow will become a required dependency of pandas:DeprecationWarning # This is due to dependencies building with a numpy version different from # the local installed numpy version, but should be fine # See https://github.com/numpy/numpy/issues/15748#issuecomment-598584838 @@ -47,3 +49,5 @@ filterwarnings = # https://github.com/pytest-dev/pytest-cov/issues/557 ignore:The --rsyncdir command line argument and rsyncdirs config variable are deprecated.:DeprecationWarning ignore:.*is deprecated and slated for removal in Python 3:DeprecationWarning + # pyparsing 3.3 raises dep warnings with old mpl in oldestdeps + ignore::pyparsing.warnings.PyparsingDeprecationWarning diff --git a/tox.ini b/tox.ini index f08d4b0..3767b19 100644 --- a/tox.ini +++ b/tox.ini @@ -3,9 +3,9 @@ min_version = 4.0 requires = tox-pypi-filter>=0.14 envlist = - py{310,311,312}{,-online,-sunpy} - py312-devdeps - py310-oldestdeps + py{312,313,314}{,-online,-sunpy} + py314-devdeps + py312-oldestdeps codestyle build_docs @@ -41,15 +41,15 @@ setenv = JSOC_EMAIL = jsoc@sunpy.org deps = # For packages which publish nightly wheels this will pull the latest nightly - devdeps: numpy>=0.0.dev0 + # devdeps: astropy>=0.0.dev0 # Packages without nightly wheels will be built from source like this # devdeps: git+https://github.com/ndcube/ndcube oldestdeps: minimum_dependencies - # The following indicates which extras_require will be installed + # Extra pluings for the CI pytest-xdist pytest-timeout # These are specific extras we use to run the sunpy tests. - sunpy: git+https://github.com/sunpy/sunpy + sunpy: sunpy>=0.0.dev0 sunpy: beautifulsoup4 sunpy: mpl-animators sunpy: reproject