-
Notifications
You must be signed in to change notification settings - Fork 0
/
astroupdate.py
703 lines (634 loc) · 30 KB
/
astroupdate.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
import subprocess
import os
import glob
import webbrowser
import sys
import json
import re
import requests
import pandas as pd
if int(pd.__version__.split('.')[0])>=1:
pd.set_option('display.max_colwidth', None)
else:
pd.set_option('display.max_colwidth', -1)
#from utils import html_to_pandas
from bs4 import BeautifulSoup
import jinja2
if sys.version_info.major >= 3:
raw_input = input
def update_astroupdate(defdir='/software/github/heasarc/astro_update/definitions',
deffile='astroupdate_defs_MASTER.json', clobber=True,
outdir="/software/github/heasarc/astro_update/html",
outname="astro-update.html",
templatedir='/software/github/heasarc/astro_update/templates'):
"""Updates the astro-update definitions file then creates the astro-update webpage
:param defdir: full path to directory where json definitions file is stored
:param deffile: name of json definitions file in <defdir> to use
:param clobber: overwrith astro-update.html file if True
:param outdir: output directory for astro-update.html file
:param outname: name of output file (default = astro-update.html)
:return: status (0 if no errors encountered)
UPDATES:
20180716: Changed lxml parser to html5lib to avoid error on heasarcdev
"""
# adj = json.load(open('{0}/astroupdate_defs.json'.format(defdir),'r')) #load as json dictionary
aud = read_au_defs(deffile=deffile, defdir=defdir)
# aud = aud.loc[['spex']]
software = aud.index.values
status = 0
for s in software:
if s == 'pros':
parser = 'html.parser'
else:
#parser = 'lxml'
parser = 'html5lib'
curvers, currel = get_au_curvers(aud, s, parser=parser)
# only update if both curvers and currel found
if curvers != "Not Found":
print ("Updating {0} Current version to {1}".format(s, curvers))
aud.loc[s].ad_version = curvers
else:
print ("Could not retrieve release version for {0}".format(s))
if curvers != "Not Found":
print ("Updating {0} Current release to {1}".format(s, currel))
aud.loc[s].ad_release_date = currel
else:
print("Could not retrieve release date for {0}".format(s))
print ("writing updated defs file")
newfile = write_newdefs(aud, defdir=defdir, clobber=True)
print ("Successfully wrote {0}".format(newfile))
# strip directory path from newfile filename to pass to make_astroupdate_page
try:
newfnameind = newfile.strip().rfind('/')
newfname = newfile[newfnameind:]
except:
print("Could not find directory marker in {0}".format('newfile'))
newfname = newfile
# now make the updated html page
try:
make_astroupdate_page(outdir=outdir,
defdir=defdir,
deffile=newfname,
outname=outname,
templatedir=templatedir,
clobber=clobber)
except Exception as errmsg:
status = -1
sys.exit('Error creating astroupdate.html; Exiting ({0})'.format(errmsg))
return status
def make_astroupdate_page(outdir="/software/github/heasarc/astro_update/html",
defdir='/software/github/heasarc/astro_update/definitions',
deffile='astroupdate_defs.json',
outname='astro-update.html',
templatedir="/software/github/heasarc/astro_update/templates",
clobber=True):
"""Creates the astro-update.html web page
Creates the astro-update.html web page using python's Jinja2 templating engine
based on the information in the ``<defdir>/<deffile>`` file
Templates::
astroupdate_table_template.html: the astro-update html table
astroupdate_template.html: base template
:param outdir: output directory where <outname> file is written
:param defdir: directory where the json definition file is stored
:param deffile: name of json definitions file in defdir to use
:param outname: name of output html file
:param templatedir: directory where jinja2 templates are stored
:param clobber: overwrite file if True, else don't overwrite
:return: status (0 = ok, -1 = exception)
"""
status = 0
aud_df = read_au_defs(defdir=defdir, deffile=deffile)
# convert release date to a datetime object...
aud_df.ad_release_date = pd.to_datetime(aud_df.ad_release_date, errors="coerce")
# ... then sort by release_date
aud_df.sort_values(by='ad_release_date', ascending=False, inplace=True)
audef_for_web = aud_df[aud_df.ad_release_date.notnull()]
# Render html file
try:
templateLoader = jinja2.FileSystemLoader(searchpath=templatedir)
except Exception as errmsg:
print ('Could not load templates in {0}; Exiting ({1})'.format(templatedir, errmsg))
status = -1
return status
templateEnv = jinja2.Environment(loader=templateLoader)
# get child template
template = templateEnv.get_template('astroupdate_table_template.html')
output_html = template.render(audef=audef_for_web)
fname = '{0}/{1}'.format(outdir, outname)
if os.path.isfile(fname):
if not clobber:
sys.exit("File {0} exists, and clobber = False; not overwritten".format(fname))
try:
with open(fname, mode='w') as fout:
fout.write(output_html)
except IOError as errmsg:
print('Could not write {0}; Exiting ({1})'.format(fname, errmsg))
status = -1
return status
print ("Wrote {0}".format(fname))
return status
def read_au_defs(defdir='/Users/mcorcora/software/github/heasarc/astro_update/definitions',
deffile='astroupdate_defs_MASTER.json'):
""" returns a pandas dataframe of astro-update software definitions
returns a pandas dataframe version of the entries in the astro-update json software definition file
:param ausoftdef: name of software definition file
:return aud_df: pandas dataframe of the astro-update software definitions
"""
ausoftdef = '{0}/{1}'.format(defdir, deffile)
audict = json.load(open(ausoftdef, mode='r'))
aud_df = pd.DataFrame.from_dict(audict, orient='index')
return aud_df
def get_au_curvers(aud_df,software, strlen=100, open_url = False, parser='lxml'):
""" gets the version for the specified software
From the astro-update dataframe (as created by read_au_defs()), gets the version for the specified software from
the software maintainer's webpage, parses software's version_url page and returns current version and current
release date of the software
:param aud_df: input astro_update dataframe (from read_au_defs)
:param sofware: input name of software pacakage (as indexed in the dataframe
:param strlen: number of characters after pattern_marker to search for version string
:param open_url: if True will open the URL containing the version information for the specified software
:param parser: html parser to be used by BeautifulSoup (default = "lxml")
:return: curver, currel = current version and current release date of software;
if current version or current release can't be determined, their value is set to 'Not Found'
"""
# initialize curver, currel to current values in the astroupdate defs file
curver = aud_df.loc[software].ad_version
currel = aud_df.loc[software].ad_release_date
#
# get current software version from version_url
#
try:
vurl = aud_df.loc[software].version_url
except KeyError as errmsg:
print(errmsg)
print("Available software keys:" )
for i in aud_df.index:
print (i,end = " ")
sys.exit("Exiting")
try:
req = requests.get(vurl)
except requests.exceptions.ConnectionError as errmsg:
print("{0}; returning".format(errmsg))
return curver, currel
softsoup = BeautifulSoup(req.text, parser)
pattern = aud_df.loc[software].pattern
pattern_mark = aud_df.loc[software].pattern_marker
if pattern_mark:
indstart = softsoup.text.lower().find(pattern_mark.lower())
indend = indstart+strlen
else:
indstart = 0
indend = -1
searchstr = softsoup.text[indstart:indend]
try:
curver = re.compile(pattern).search(searchstr).group()
except AttributeError as errmsg:
print("Could not find pattern {0} in text at {2} ({3})".format(pattern, searchstr, vurl, errmsg))
if open_url:
webbrowser.open(vurl)
#
# Get release date
#
# first get the soup of the page which has the release date
try:
rurl = aud_df.loc[software].release_url
except KeyError as errmsg:
print(errmsg)
print("Available software keys:" )
for i in aud_df.index:
print (i, end= " ")
sys.exit("Exiting")
try:
req = requests.get(rurl)
except requests.exceptions.ConnectionError as errmsg:
print("{0}; returning".format(errmsg))
return curver, currel
softsoup = BeautifulSoup(req.text, parser)
rel_pattern = aud_df.loc[software].release_pattern
rel_mark = aud_df.loc[software].release_marker
# if the rel_mark is an empty string, use the current version string as the rel_mark
if not rel_mark:
rel_mark = curver
indstart = softsoup.text.lower().find(rel_mark.lower())
indend = indstart+strlen
searchstr = softsoup.text[indstart:indend]
# the following two substitutions are needed to decode release date
# for sherpa (first remove newline then multiple spaces)
searchstr = re.sub('\n', ' ',searchstr)
searchstr = re.sub('\s+', ' ', searchstr)
newrel = "Not Found"
try:
newrel = re.compile(rel_pattern).search(searchstr).group()
except AttributeError as errmsg:
print("Could not find pattern {0} in text at {2} ({3})".format(rel_pattern, searchstr, vurl, errmsg))
if open_url:
webbrowser.open(vurl)
if newrel != "Not Found":
currel = parse_reldate(newrel)
return curver.strip(), currel.strip()
def astroupdate_dict(url="http://heasarc.gsfc.nasa.gov/docs/heasarc/astro-update/"):
"""retrieve astroupdate dictionary
Returns a dictionary of software packages monitored by Astro-Update based on the contents of the specified
astro-update html file
:param url: astro-update URL from which dictionary will be obtained
:return: astro-update python dictionary
"""
from bs4 import BeautifulSoup
#import urllib2
#response = urllib2.urlopen('http://heasarc.gsfc.nasa.gov/docs/heasarc/astro-update/')
#html = response.read()
html = requests.get('http://heasarc.gsfc.nasa.gov/docs/heasarc/astro-update/')
soup = BeautifulSoup(''.join(html.text), 'lxml')
table = soup.findAll('table')
soft_table = table[1] # there are 3 tables on the page, the software version table is the 2nd table
rows = soft_table.findAll('tr')
au_dict = dict()
for row in rows[1:]:
cols = row.findAll('td')
n = ''.join(cols[0].find(text=True))
r = ''.join(cols[1].find(text=True))
v = ''.join(cols[2].find(text=True))
m = ''.join(cols[3].find(text=True))
homepage = cols[0].findAll('a')[0]['href']
download = cols[2].findAll('a')[0]['href']
au_dict[str(n).lower().strip()] = {'Version': str(v).strip(),
'Date': str(m).strip(), 'Author': str(r).strip(),
'HomePage': homepage, 'Download': download}
return au_dict
def astroupdate(software, chatter=0):
"""return the current astro-update version and other info for specified software
If a specified software package is monitored in astro-update,
this will return the current astro-update version, date of last update, the
software author, and the url to download the latest update,
from the astro-updated web dictionary returned by astroupdate_dict()
:param software: name of software package to check (as given by the astro-update json definitions file)
:param chatter: verbosity
:return: dictionary entry of software information
"""
aud=astroupdate_dict()
softkey=software.strip().lower()
try:
aud[softkey]
except KeyError:
print ("{0} not monitored by Astro-Update".format(software))
print ("Valid entries are:")
print (' '.join(aud.keys()))
return 0
if chatter > 0:
print ("{0} was last updated to version {1} on {2} by {3}".format(software,
aud[softkey]['Version'],
aud[softkey]['Date'],
aud[softkey]['Author']))
"""
ver = str(aud[softkey]['Version'])
date = str(aud[softkey]['Date'])
author = str(aud[softkey]['Author'])
updateurl = str(aud[softkey]['URL'])
return ver, date, author, updateurl
"""
return aud[softkey]
def auto_update(software):
"""notify the user if their installed version of specified software is up-to-date
For the specified software, notify the user if their installed version is up-to-date; if not,
give the user the option of downloading the latest version (by taking the user to the
software download page in their web browser).
:param software: name of software to check
:return: (no output)
"""
ad = astroupdate_dict()
software = software.strip().lower()
try:
current_vers = ad[software]['Version']
except:
print ("{0} not found in Astro-Update Database; returning".format(software))
return
updateurl = ad[software]["URL"]
"""
Compare installed version of software to most recent version
"""
if software=="heasoft":
headasdir = os.getenv('HEADAS')
if not headasdir:
print ("HEASoft not configured (environment variable $HEADAS is not defined); stopping")
return
try:
fver_installed = subprocess.check_output(['fversion'])
except:
print ("fversion failed; is HEASoft installed?")
sys.exit()
vers=fver_installed.strip("\n").split('_V')
vers=vers[1].rstrip()
print ("Latest version of HEASoft = {0}; You currently have HEASoft version {1}".format(current_vers, vers))
if current_vers.strip() != vers.strip():
ans=''
ans=raw_input("Would you like to update (Y/n)? ")
if ans.strip()=='' or ans[0].lower()=='y':
print ("Opening HEASoft download page in your web browser")
#webbrowser.open('http://heasarc.gsfc.nasa.gov/docs/software/lheasoft/download.html')
webbrowser.open(updateurl)
if software=='sae':
fermidir = os.getenv('FERMI_DIR')
if not fermidir:
print ("SAE not configured (environment variable $FERMI_DIR is not defined); stopping")
return
STools = glob.glob(fermidir+'/../../ScienceTools*') # Find Science Tools directory
#print STools
foundversion = False
if not STools:
print ("Problem finding {0}; Is SAE installed?".format(fermidir+'/../..'))
return
vers = STools[0].split('Tools-')[1].split('-fssc')[0].strip()
#print SToolsver
print ("The current version of the Science Analysis Environment for Fermi is {0}; you have version {1}".format(current_vers.strip(), vers.strip()))
if current_vers.strip() != vers.strip():
ans=raw_input("Would you like to update (Y/n)? ")
if ans.strip()=='' or ans[0].lower()=='y':
print ("Opening Fermi SAE download page in your web browser")
#webbrowser.open('http://fermi.gsfc.nasa.gov/ssc/data/analysis/software/')
webbrowser.open(updateurl)
if software =='xspec':
headasdir = os.getenv('HEADAS')
foundversion = False
if not headasdir:
print ("XSPEC not configured (environment variable $HEADAS is not defined); stopping")
return
"""
Get installed primary version number from locally installed manual.html; this is defined as a constant in
the file $HEADAS/../Xspec/src/XSUtil/Utils/XSutility.cxx in a line like this: static const string version = "12.9.0b";
"""
xsutil=headasdir+'/../Xspec/src/XSUtil/Utils/XSutility.cxx'
f=open(xsutil,'r')
for line in f.readlines():
#print line
if 'version =' in line:
#print line
vers = line.split('=')[1].strip().split('"')[1]
#print 'Vers = {0}'.format(vers.strip())
foundversion=True
break
f.close()
if not foundversion:
print ("Could not find local XSpec version; stopping")
return
print ("The current version of XSpec is {0}; you have version {1}".format(current_vers.strip(), vers.strip()))
if current_vers.strip() != vers.strip():
ans=raw_input("Would you like to update (Y/n)? ")
if ans.strip()=='' or ans[0].lower()=='y':
print ("Opening the XSpec download page in your web browser")
#webbrowser.open('http://fermi.gsfc.nasa.gov/ssc/data/analysis/software/')
webbrowser.open(updateurl)
if software == 'ciao':
ascds = os.getenv('ASCDS_INSTALL')
if not ascds:
print ("CIAO not configured (environment variable $ASCDS_INSTALL is not defined); stopping")
return
os.system(ascds+'/contrib/bin/check_ciao_version')
if software == 'sas':
sasdir = os.getenv('SAS_DIR')
if not sasdir:
print ("SAS not configured (environment variable $SAS_DIR is not defined); stopping")
return
vers = sasdir.split('sas_')[1].split('/')[0]
print ("The current version of SAS is {0}; you have version {1}".format(current_vers.strip(), vers.strip()))
if current_vers.strip() != vers.strip():
ans = raw_input("Would you like to update (Y/n)? ")
if ans.strip() == '' or ans[0].lower() == 'y':
print ("Opening SAS download page in your web browser")
# webbrowser.open('http://fermi.gsfc.nasa.gov/ssc/data/analysis/software/')
webbrowser.open(updateurl)
return
def aud_check(url="https://heasarc.gsfc.nasa.gov/docs/heasarc/astro-update/astro-update.html", soft=""):
"""A visual inspection of software monitored by astro-update (**Deprecated**)
This function checks the speficied astro-update web page given by <url> and displays the version page
so that the user can visually check that the version on the astro-update page is current. **This routine has been
deprecated**.
:param url: the (usually) public url of the astro-update web page
:param soft: an array of software titles (for eg: ['heasoft']) that have out of date versions
:return:
"""
aud = astroupdate_dict(url=url)
need_to_update=[]
# if soft is not defined, check all the softwared monitored by astro-update
if not soft:
soft=aud.keys()
for s in soft:
try:
version = aud[s.lower()]['Version']
except:
sys.exit('Could not get version for {0}; exiting'.format(s.lower()))
print ("Astro-Update Version of {0} = {1}".format(s,version))
print ("Displaying Version Page for {0}".format(s))
webbrowser.open(aud[s.lower()]['HomePage'])
ans = raw_input('Enter Version from software version page (return to ignore) > ')
if len(ans.strip()) >= 1:
need_to_update.append((s.lower(), ans))
return need_to_update
def aud_init_defs(software, aud_url="http://heasarc.gsfc.nasa.gov/docs/heasarc/astro-update/",
fdir='/software/github/heasarc/astro_update/definitions/TEST', clobber=True):
""" Initialize the astro-update definitions file (**Deprecated**)
Creates the initial astro-update definitions file for software listed on the astro-update page.
This was written to help create the initial json definitions file but is no longer needed
:param software: name of software to be added (Mixed case allowed)
:param aud_url: url of the astro-update file
:param fdir: file output directory
:return: (no return value)
"""
req = requests.get(aud_url)
ausoup = BeautifulSoup(req.text,'lxml')
tab = ausoup.findAll('table')
rows = tab[1].findAll('tr')
# create list of all astro-update software names in lower case
auname = [x.findAll('td')[0].text.split(':')[0].strip().lower() for x in rows[1:]]
try: # see if software already exists in astro-update table
ind = auname.index(software.strip().lower())
ind = ind+1 # add one to offset from header
r = rows[ind]
col = r.findAll('td')
name = str(col[0].text.split(':')[0].strip().lower())
desc = str(col[0].text.split(':')[1]).strip()
homepage = str(col[0].a.get('href'))
auth = str(col[1].text)
ver = str(col[2].text)
download = str(col[2].a.get('href'))
updated = str(col[3].text)
# note - json needs strings to have double, not single, quotes; get around this through use of json.dump()
audict = {"{0}".format(name): {"pattern": "\\d+\\.{0,1}\\d+",
"pattern_description": "one or more digits followed by zeror or 1 period followed by one or more digits",
"pattern_marker": "version",
"home_url": "{0}".format(homepage),
"download_url": "{0}".format(download),
"version_url": "{0}".format(download),
"description": "{0}".format(desc),
"author": "{0}".format(auth),
"author_page": "{0}".format(homepage),
"version":"{0}".format(ver),
"updated":"{0}".format(updated)}}
fname = "{0}/{1}_au.json".format(fdir, name)
create_file = True
if os.path.isfile(fname):
if not clobber:
sys.exit("File {0} exists, and clobber = False; not overwritten".format(fname))
else:
try:
with open(fname, mode='w') as fout:
json.dump(audict, fout, indent=4)
except IOError as errmsg:
print (errmsg)
except ValueError:
ans = raw_input("{0} NOT FOUND IN ASTROUPDATE TABLE; create new file? [y]/n")
if ans.strip()[0].lower() != 'n':
audict = {"{0}".format(name): {"pattern": "\\d+\\.{0,1}\\d+",
"pattern_description": "one or more digits followed by zeror or 1 period followed by one or more digits",
"pattern_marker": "version",
"home_url": "{0}".format('ENTER HOME URL'),
"download_url": "{0}".format('ENTER DOWNLOAD URL'),
"version_url": "{0}".format('ENTER VERSION URL'),
"description": "{0}".format('ENTER DESCRIPTION'),
"author": "{0}".format('ENTER AUTHOR'),
"author_page": "{0}".format('ENTER AUTHOR URL')}}
if os.path.isfile(fname):
if not clobber:
sys.exit("File {0} exists, and clobber = False; not overwritten".format(fname))
else:
try:
with open(fname, mode='w') as fout:
json.dump(audict, fout, indent=4)
except IOError as errmsg:
sys.exit(errmsg)
return
def aud_table_update(software, new_version, update_date, aud_url = 'http://heasarc.gsfc.nasa.gov/docs/heasarc/astro-update'):
""" Turns astro-update html software table to a BeautifuSoup table object (**Deprecated?**)
Gets the astro-update table, updates version of software and date of update, then returns a BeautifulSoup table object
with updated information (**deprecated?**)
:param software: software package to update (eg. heasoft)
:param new_version: version number of latest update
:param update_date: data of new update
:param aud_url: URL of astro-update
:return:
"""
# get html with requests
# req = requests.get(aud_url)
# create soup and extract table
# audtable = BeautifulSoup(req.text, 'lxml')('table')[1] # software table is table 1 on the page
# TODO: update row or add new row
#
# then create dataframe
auddf = pd.read_html(aud_url, attrs={'id':'astroupdate_software'})
auddf = auddf[0] # read_html seems to return a list so select 1st element
# sort dataframe by time of last update, descending
# first create sort column called sortdate
auddf['sortdate'] = pd.to_datetime(auddf['Last Update'])
# now sort table by sortdate column, descending
auddf.sort_values(by="sortdate", ascending=False, inplace=True)
# create html using beautifulsoup
soup = BeautifulSoup(auddf.to_html(escape=False), 'lxml')('table')[0]
#print(soup.prettify()) print auddfsort.head()
return soup
def parse_reldate(date):
"""parses the release date
Parses the release date using dateutil to convert to the astro-update standard format (MM/DD/YYYY)
**I believe this is not complete and could be removed**
:param date: release date of form understood by dateutil.parser
:return: TODO: Date object (or maybe a formatted date string?)
"""
from dateutil.parser import parse
try:
dateobj = parse(date)
datestr = dateobj.strftime('%m/%d/%y')
except:
datestr = date
return datestr
def write_newdefs(aud_df, outroot='astroupdate_defs',
defdir='/software/github/heasarc/astro_update/definitions',
clobber = False):
""" Creates json astro-update.defs file updated with current software version/release date
Outputs the astro-update dataframe as a json astro-update definitions file
:param aud_df: astro-update dataframe
:param outroot: file rootname (default = 'astroupdate_defs')
:param defdir: file ouptput directory
:param clobber: if True, will overwrite an existing file
:return: name of output file
"""
import datetime
now = datetime.datetime.now()
fname = "{0}/{1}_{2}.json".format(defdir,outroot, now.strftime('%Y%m%d'))
if os.path.isfile(fname):
if not clobber:
sys.exit("{0} exists and clobber = False; exiting".format(fname))
newdef = open(fname, 'w')
adj = json.loads(aud_df.to_json(orient='index'))
json.dump(adj, newdef, indent=4)
return fname
def main():
sinfodir = '/software/github/ipython_notebooks/resources/'
sinfo = json.load(open('{0}/astro_update_sinfo.txt'.format(sinfodir)))
sinfo.keys()
keys = sinfo.keys()
for k in keys:
print (k)
pat = re.compile(sinfo[k]['pattern'])
aud = astroupdate_dict()
req = requests.get(aud[k]['Download'])
skey = sinfo[k]['version_marker']
skip = False
try:
ind = req.text.index(skey)
except:
print ("{0} not found on {1}".format(skey, aud[k]['Download']))
skip = True
if not skip:
req.text[ind:]
ver = pat.search(req.text[ind:ind + 100])
req.close()
try:
print ("Current version of the {1} is {0}; expected {2}".format(ver.group(0), k, aud[k]['Version']))
except:
print ("Version for {0} not found on {1}".format(k, aud[k]['Download']))
def main_check():
n2update = aud_check(url="http://heasarc.gsfc.nasa.gov/docs/heasarc/astro-update/", soft=["HEASOFT"])
print (n2update)
def main_curvers(software, parser='lxml'):
aud_df = read_au_defs(deffile='astroupdate_defs_test.json')
curvers = get_au_curvers(aud_df, software, parser=parser)
print ("\ncurrent version of {0} is {1}".format(aud_df.loc[software]['name'], curvers))
def main_curvers_complete():
aud = read_au_defs()
software = aud.index.values
for soft in software:
if soft == 'pros':
parser = 'html.parser'
else:
parser = 'lxml'
curvers = get_au_curvers(aud, soft, parser=parser)
print ("\ncurrent version of {0} is {1} (released {2})".format(soft, curvers[0], curvers[1]))
def main_update_ad(defdir='/software/github/heasarc/astro_update/definitions'):
#adj = json.load(open('{0}/astroupdate_defs.json'.format(defdir),'r')) #load as json dictionary
aud = read_au_defs()
#aud = aud.loc[['spex']]
software = aud.index.values
for s in software:
if s == 'pros':
parser = 'html.parser'
else:
parser = 'lxml'
curvers, currel = get_au_curvers(aud, s, parser=parser)
# only update if both curvers and currel found
if curvers != "Not Found" and currel != "Not Found":
print ("Updating {0} Current version to {1}".format(s, curvers))
aud.loc[s].ad_version = curvers
print ("Updating {0} Current release to {1}".format(s, currel))
aud.loc[s].ad_release_date = currel
else:
print("Could not retrieve versions for {0}".format(s))
print ("writing updated defs file")
newfile = write_newdefs(aud, clobber=True)
print ("Successfully wrote {0}".format(newfile))
if __name__ == '__main__':
#main()
#main_check()
#main_curvers('pros', parser='html.parser')
main_curvers('heasoft')
#main_curvers_complete()
#main_update_ad()
#make_astroupdate_page()
#update_astroupdate()