-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #47 from karan2704/mseed
Append ts segments to m3u8 file
- Loading branch information
Showing
16 changed files
with
464 additions
and
104 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,6 +9,7 @@ MAINTAINER Orcasound <[email protected]> | |
|
|
||
| RUN pip3 install numpy | ||
| RUN pip3 install obspy | ||
| RUN pip3 install ooipy | ||
|
|
||
| ############################### Copy files ##################################### | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
|
|
||
| date=$(date '+%Y-%m-%d') | ||
|
|
||
| mkdir -p /tmp/$NODE_NAME | ||
| mkdir -p /tmp/$NODE_NAME/hls | ||
| mkdir -p /tmp/$NODE_NAME/hls/$date | ||
|
|
||
| echo "starting upload" | ||
|
|
||
| python3 upload_s3.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,37 +1,26 @@ | ||
| version: "3" | ||
| services: | ||
| pull: | ||
| fetch: | ||
| image: orcastream/orcamseed | ||
| build: ./ | ||
| # command: tail -F README.md | ||
| command: python3 mseedpull.py | ||
| command: python3 ooipypull.py | ||
| restart: always | ||
| env_file: .env | ||
| volumes: | ||
| - data:/root/data | ||
| stream: | ||
| - data:/tmp | ||
| depends_on: | ||
| - upload | ||
|
|
||
| upload: | ||
| image: orcastream/orcamseed | ||
| build: ./ | ||
| # command: tail -F README.md | ||
| command: ./streamfiles.sh | ||
| command: tail -F README.md | ||
| restart: always | ||
| env_file: .env | ||
| volumes: | ||
| - data:/root/data | ||
| - data:/tmp | ||
| privileged: true | ||
| logspout: | ||
| image: gliderlabs/logspout | ||
| command: ${SYSLOG_URL} | ||
| restart: always | ||
| hostname: ${NODE_NAME} | ||
| env_file: .env | ||
| environment: | ||
| - SYSLOG_HOSTNAME=${NODE_NAME} | ||
| volumes: | ||
| - /var/run/docker.sock:/var/run/docker.sock | ||
| ports: | ||
| - "8000:8000" | ||
|
|
||
|
|
||
|
|
||
| volumes: | ||
| data: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| import ooipy | ||
| import os | ||
| import datetime | ||
| import shutil | ||
| import logging | ||
| import logging.handlers | ||
| import sys | ||
|
|
||
| LOGLEVEL = logging.DEBUG | ||
| PREFIX = os.environ["TIME_PREFIX"] | ||
| DELAY = os.environ["DELAY_SEGMENT"] | ||
| NODE = os.environ["NODE_NAME"] | ||
| BASEPATH = os.path.join('/tmp', NODE) | ||
| PATH = os.path.join(BASEPATH, 'hls') | ||
|
|
||
| log = logging.getLogger(__name__) | ||
|
|
||
| log.setLevel(LOGLEVEL) | ||
|
|
||
| handler = logging.StreamHandler(sys.stdout) | ||
|
|
||
| formatter = logging.Formatter('%(module)s.%(funcName)s: %(message)s') | ||
| handler.setFormatter(formatter) | ||
|
|
||
| log.addHandler(handler) | ||
|
|
||
| def fetchData(start_time, segment_length, end_time, node): | ||
| os.makedirs(BASEPATH, exist_ok=True) | ||
| os.makedirs(PATH, exist_ok=True) | ||
| while start_time < end_time: | ||
| segment_end = min(start_time + segment_length, end_time) | ||
|
|
||
| #paths and filenames | ||
| datestr = start_time.strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3] | ||
| sub_directory = start_time.strftime("%Y-%m-%d") | ||
| file_path = os.path.join(PATH, sub_directory) | ||
| wav_name = "{date}.wav".format(date=datestr) | ||
| ts_name = "{prefix}{date}.ts".format(prefix=PREFIX, date=datestr) | ||
|
|
||
| #create directory and edit latest.txt | ||
| if not os.path.exists(file_path): | ||
| os.mkdir(file_path) | ||
| manifest_file = os.path.join('/root', 'live.m3u8') | ||
| if os.path.exists(manifest_file): | ||
| os.remove(manifest_file) | ||
| if not os.path.exists(os.path.join(BASEPATH, 'latest.txt')): | ||
| with open('latest.txt', 'x') as f: | ||
| f.write(sub_directory) | ||
| shutil.move('latest.txt', BASEPATH ) | ||
| else: | ||
| with open(f'/{BASEPATH}/latest.txt', 'w') as f: | ||
| f.write(sub_directory) | ||
|
|
||
|
|
||
| #fetch if file doesn't already exist | ||
| if(os.path.exists(os.path.join(file_path, ts_name))): | ||
| print("EXISTS") | ||
| continue | ||
| hydrophone_data = ooipy.request.hydrophone_request.get_acoustic_data( | ||
| start_time, segment_end, node, verbose=True, data_gap_mode=2 | ||
| ) | ||
| if hydrophone_data is None: | ||
| print(f"Could not get data from {start_time} to {segment_end}") | ||
| start_time = segment_end | ||
| continue | ||
| print(f"data: {hydrophone_data}") | ||
| hydrophone_data.wav_write(wav_name) | ||
|
|
||
| writeManifest(wav_name, ts_name) | ||
|
|
||
| #move files to tmp for upload | ||
| shutil.move(os.path.join('/root', ts_name), os.path.join(file_path, ts_name)) | ||
| shutil.copy('/root/live.m3u8', os.path.join(file_path, 'live.m3u8')) | ||
| os.remove(wav_name) | ||
| start_time = segment_end | ||
|
|
||
|
|
||
| def writeManifest(wav_name, ts_name): | ||
| root_path = os.path.join('/root', 'live.m3u8') | ||
| if not os.path.exists(root_path): | ||
| os.system("ffmpeg -i {wavfile} -f segment -segment_list 'live.m3u8' -strftime 1 -segment_time 300 -segment_format mpegts -ac 1 -acodec aac {tsfile}".format(wavfile=wav_name, tsfile=ts_name)) | ||
| else: | ||
| os.system('ffmpeg -i {wavfile} -f mpegts -ar 64000 -acodec aac -ac 1 {tsfile}'.format(wavfile=wav_name, tsfile=ts_name)) | ||
| #remove EXT-X-ENDLIST and write new segment | ||
| with open("live.m3u8", "r+") as f: | ||
| lines = f.readlines() | ||
| f.seek(0) | ||
| f.truncate() | ||
| f.writelines(lines[:-1]) | ||
| f.write("#EXTINF:300.000000, \n") | ||
| f.write(f'{ts_name} \n') | ||
| f.write(f'#EXT-X-ENDLIST \n') | ||
|
|
||
| #os.system("ffmpeg -i {wavfile} -hls_playlist_type event -strftime 1 -hls_segment_type mpegts -ac 1 -acodec aac -hls_segment_filename {tsfile} -hls_time 1800 -hls_flags omit_endlist+append_list live.m3u8".format(wavfile=wav_name, tsfile=ts_name)) | ||
|
|
||
|
|
||
|
|
||
| def _main(): | ||
|
|
||
| segment_length = datetime.timedelta(minutes = 5) | ||
| fixed_delay = datetime.timedelta(hours=8) | ||
|
|
||
| while True: | ||
| end_time = datetime.datetime.utcnow() | ||
| start_time = end_time - datetime.timedelta(hours=8) | ||
|
|
||
| #near live fetch | ||
| fetchData(start_time, segment_length, end_time, 'PC01A') | ||
|
|
||
| #delayed fetch | ||
| fetchData(end_time-datetime.timedelta(hours=24), segment_length, end_time, 'PC01A') | ||
|
|
||
| start_time, end_time = end_time, datetime.datetime.utcnow() | ||
|
|
||
|
|
||
|
|
||
| _main() |
Oops, something went wrong.