diff --git a/.gitignore b/.gitignore index 25e13c9..36f14f9 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,20 @@ .idea/**/gradle.xml .idea/**/libraries +# Visual Studio Code +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + # CMake cmake-build-debug/ diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..eabd1a7 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": false + } + ] +} \ No newline at end of file diff --git a/setup.py b/setup.py index e8efd26..453f32e 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ long_description=long_description, long_description_content_type='text/markdown', entry_points={'console_scripts': ['swinfo=swiftbat.swinfo:main']}, - install_requires = ['beautifulsoup4', 'pyephem', 'astropy', 'astroquery', 'numpy'], + install_requires = ['beautifulsoup4', 'pyephem', 'astropy', 'astroquery', 'numpy', 'requests'], classifiers=['Development Status :: 3 - Alpha', 'Intended Audience :: Science/Research', 'Programming Language :: Python', 'Programming Language :: Python :: 3', 'Topic :: Scientific/Engineering :: Astronomy', ], diff --git a/swiftbat/__init__.py b/swiftbat/__init__.py index 31573ee..d4cc5f6 100644 --- a/swiftbat/__init__.py +++ b/swiftbat/__init__.py @@ -19,8 +19,9 @@ from __future__ import print_function, division, absolute_import +from .generaldir import * + from .clockinfo import utcf from .swutil import * from .swinfo import * from .batcatalog import BATCatalog -from . import generaldir diff --git a/swiftbat/clockinfo.py b/swiftbat/clockinfo.py index a764ceb..a453255 100644 --- a/swiftbat/clockinfo.py +++ b/swiftbat/clockinfo.py @@ -6,7 +6,7 @@ import datetime import re from pathlib import Path -from .generaldir import httpDir +from .generaldir import HTTPDir """ From the FITS file: @@ -118,7 +118,7 @@ def updateclockfiles(self, clockdir, ifolderthan_days=30, test_days=1): except: pass try: - clockremotedir = httpDir(self.clockurl) + clockremotedir = HTTPDir(self.clockurl) clockremote = sorted(clockremotedir.getMatches("", self.clockfile_regex))[-1] locallatest = Path(clockdir).joinpath(Path(clockremote).name) clockremotedir.copyToFile(clockremote, locallatest) diff --git a/swiftbat/data.py b/swiftbat/data.py new file mode 100644 index 0000000..49cb036 --- /dev/null +++ b/swiftbat/data.py @@ -0,0 +1,115 @@ +""" +Find and manage Swift data files from Heasarc +""" + +from astroquery.heasarc import Heasarc +import astropy as ap +import datetime +from swiftbat.swutil import any2datetime, datetime2mjd +# import requests +from functools import lru_cache +from pathlib import Path +# import urllib +from swiftbat.generaldir import HTTPDir + +@lru_cache +def heasarc_tdrss_dir(): + return HTTPDir("https://heasarc.gsfc.nasa.gov/FTP/swift/data/tdrss") + +def _heasarc_TTE_location(trignum:int, time:datetime.datetime, matchstring="bevshsp_uf.evt"): + """The URL for time-tagged-event data for a specific trigger + trignum: Trigger number (or TARGET_ID) + time: datetime of trigger + matchstring: must be a straight string or a python regex (i.e. not a shell wildcard string) found in filename + """ + time = time.replace(tzinfo=None) # Make it naive + monthstring = f"{time:%Y_%m}" + tdrssdir = heasarc_tdrss_dir() + try: + subdir = f"{monthstring}/{trignum:08d}000" + matches = tdrssdir.matchPath(subpath=subdir, wildpath=f'.*{matchstring}.*') + # tdrsslist = sftp.listdir(subdir) + # for fname in tdrsslist: + # if matchstring in fname: + # return (swiftdataurl, subdir, fname) + except Exception as e: + pass + age = datetime.datetime.utcnow() - time + + + +def heasarcquery(cache=True, **queryargs): + heasarc = Heasarc() + if False: + queryargs.setdefault('position','0.0d 0.0d') + queryargs.setdefault('radius','361 deg') + payload = heasarc.query_region(get_query_payload=True, **queryargs) + for field in ['Entry', 'Coordinates', 'Radius']: + payload.pop(field) + result = heasarc.query_async(payload, cache=cache) + result = heasarc._parse_result(result) + else: + object_name = queryargs.pop('object_name', None) + result = heasarc.query_object(object_name=object_name, cache=cache, **queryargs) + return result + + +def triggerrecords(time=None, timewindow_d=1/24, trignum=None, cache=True, **queryargs): + """Get trigger records from the 'swifttdrss' table of HEASARC + + Table definition: + https://heasarc.gsfc.nasa.gov/W3Browse/all/swifttdrss.html + Do your own Quality Control on the results + + time can be a single value (which will be extended by timewindow_d days + on each side) or a list, where the extreme values will be used as a range + trignum can be a single value, or a list where the extreme values + will be used as an inclusive range + + time gets converted to an MJD to match the 'TIME' column of the result + trignum goes to the TARGET_ID column. + + Any other arguments are passed to the query + """ + if not queryargs.pop('force', False): + if trignum is None and time is None and len(queryargs) == 0: + raise RuntimeError("If you want the full table, set force=True") + + queryargs.setdefault('mission', 'swifttdrss') + queryargs.setdefault('fields', 'all') + # queryargs.setdefault('entry', '') + if time is not None: + time = any2datetime(time) + if not isinstance(time, list): + time = [time] # Start and end + mjds = [datetime2mjd(t) for t in time] + if len(mjds) == 1: + mets = [mjds[0] - timewindow_d, + mjds[0] + timewindow_d] + queryargs['TIME'] = f"{min(mjds)} .. {max(mjds)}" + if trignum is not None: + try: + trignum = [tr_ for tr_ in trignum] + except TypeError: + trignum = trignum + pass + queryargs['target_id'] = (f"{trignum[0]}" if len(trignum) == 1 + else f"{min(trignum)} .. {max(trignum)}") + return heasarcquery(cache=cache, **queryargs) + +def main_test(): + import swiftbat + result_trigrange = triggerrecords(trignum=[1104735, 1104873]) + result_timerange = triggerrecords(time=["2022-05-01", "2022-05-02"]) + # Look at one row + r = result_trigrange[2] + location = _heasarc_TTE_location(r['TARGET_ID'], swiftbat.mjd2datetime(r['TIME'])) + + timechecks = [(swiftbat.mjd2datetime(r['TIME']) - swiftbat.met2datetime(r['TIME_SECONDS'], correct=True)).total_seconds() + for r in result_trigrange] + + pass + +if __name__ == '__main__': + main_test() + \ No newline at end of file diff --git a/swiftbat/generaldir.py b/swiftbat/generaldir.py index 511bfb7..1dd20fd 100644 --- a/swiftbat/generaldir.py +++ b/swiftbat/generaldir.py @@ -43,7 +43,7 @@ def subTemplate(origstring, keydict, return s -class generalDir: +class GeneralDir: _rewildsplitter = re.compile(r'''(?P(/*[^$/]+[/]+)*)(?P[^/]*)/*(?P.*)''') def __init__(self, url): @@ -100,9 +100,11 @@ def matchPath(self, subpath, wildpath, matchdict=None, regexdict=None): """ # print("subpath = ",subpath) if subpath == None: subpath = '' + if matchdict is None: matchdict = {} + if regexdict is None: regexdict = {} (unwild, firstwild, restwild) = self._splitAndSub(wildpath, matchdict, regexdict) # print("join (%s,%s)" % (subpath,unwild)) - subpath = os.path.join(subpath, unwild) # Add the non-wild stuff to the subpath + subpath = os.path.join(subpath, unwild).rstrip('/') # Add the non-wild stuff to the subpath # print("->",subpath) if not self.exists(subpath): # if the unwild stuff doesn't exist then this is a dead end # print("Deadend : %s / %s" % (self.url,subpath)) @@ -147,8 +149,8 @@ def matchPath(self, subpath, wildpath, matchdict=None, regexdict=None): def _splitAndSub(self, wildpath, matchdict, regexdict): """ returns a (topUNwildpath, firstwildness, restofwildpath) tuple with matches and regexes subbed in """ wildpath = subTemplate(wildpath, matchdict) # Do all the matching you can - if -1 == wildpath.find("$"): - return (wildpath, '', '') + # if -1 == wildpath.find("$"): + # return (wildpath, '', '') m = self._rewildsplitter.match(wildpath) # print(wildpath+"-->",m.groupdict()) if not m or (not m.groupdict()['firstwild'] and m.groupdict()['restwild']): @@ -159,7 +161,7 @@ def _splitAndSub(self, wildpath, matchdict, regexdict): return (m.groupdict()['unwild'], m.groupdict()['firstwild'], m.groupdict()['restwild']) -class httpDir(generalDir): +class HTTPDir(GeneralDir): """ HTTP specialization for the directory generalization Works on apache servers with autoindex generation A directory is read by reading from its URL with a trailing /. (Reading without @@ -184,7 +186,7 @@ class httpDir(generalDir): def __init__(self, url): if not self.validurlmatch.search(url): raise RuntimeError("Not an http url: %s" % url) - generalDir.__init__(self, url) + GeneralDir.__init__(self, url) self.filecache = {} self.dircache = {} # print("HTTP works on %s", url) @@ -213,7 +215,7 @@ def getFullPath(self, urlrelativepathname): return os.path.join(self.url, urlrelativepathname) -class ftpDir(generalDir): +class FTPDir(GeneralDir): _filelinematch = re.compile(r'''(?<=^-[r-][w-].[r-][w-].r..\s).+$''', re.MULTILINE | re.IGNORECASE) _dirlinematch = re.compile(r'''(?<=^d[r-][w-].[r-][w-].r..\s).+$''', re.MULTILINE | re.IGNORECASE) _namematch = re.compile("\S+\s*$") # last string of nonwhite characters before eol @@ -221,7 +223,7 @@ class ftpDir(generalDir): def __init__(self, url): if not re.compile("ftp://", re.IGNORECASE).search(url): raise RuntimeError("Not a valid ftp url: %s" % url) - generalDir.__init__(self, url) + GeneralDir.__init__(self, url) def files(self, subdir=""): lines = self.getMatches(subdir + '/', self._filelinematch) @@ -232,7 +234,7 @@ def dirs(self, subdir=""): return [self._namematch.findall(l.strip())[0] for l in lines] -class localDir(generalDir): +class LocalDir(GeneralDir): def __init__(self, url): # print("Trying %s as local file" % url) nofileurl = re.compile(r'''(?<=FILE://)([^>]*)''', re.IGNORECASE).findall(url) @@ -240,7 +242,7 @@ def __init__(self, url): self.dirname = os.path.realpath(nofileurl[0]) else: self.dirname = os.path.realpath(url) - generalDir.__init__(self, "FILE://" + self.dirname) + GeneralDir.__init__(self, "FILE://" + self.dirname) def dirs(self, subdir=""): d = os.path.join(self.dirname, subdir) @@ -295,7 +297,7 @@ def getFullPath(self, urlrelativepathname): def getDir(url): - for trialclass in (httpDir, ftpDir, localDir): + for trialclass in (HTTPDir, FTPDir, LocalDir): # print("trying ", trialclass) try: return trialclass(url) diff --git a/swiftbat/swinfo.py b/swiftbat/swinfo.py index a65c82b..5cff061 100755 --- a/swiftbat/swinfo.py +++ b/swiftbat/swinfo.py @@ -277,7 +277,7 @@ def updateTLE(self): try: if verbose: print("Trying to get TLE from %s" % (url_month,)) - httpdir = generaldir.httpDir(url_month) + httpdir = generaldir.HTTPDir(url_month) obsmonth = httpdir.dirs() obsmonth.reverse() for obs in obsmonth: diff --git a/swiftbat/swutil.py b/swiftbat/swutil.py index 77ac2e7..1eadc6d 100755 --- a/swiftbat/swutil.py +++ b/swiftbat/swutil.py @@ -2,6 +2,8 @@ from __future__ import print_function, division, absolute_import +from numpy import isin + """ swutil Utilities for dealing with Swift Data @@ -93,6 +95,7 @@ yearDOYsecstimeformat = r"%Y_%j_%q" # %q -> SOD in 00000 (non-standard) def any2datetime(arg, correct=True, mjd=False, jd=False): + """Change the argument into a (naive) datetime Understands: