forked from beelabhmc/flower_map
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathfind_images_upload.py
230 lines (174 loc) · 14.1 KB
/
find_images_upload.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
#!/usr/bin/env python3
import argparse
from pathlib import Path
import pandas as pd
from subprocess import Popen, PIPE
# find_images('/mnt/biology/donaldson/tom/flower_map_new/2017_6217East.csv',
# '/mnt/biology/donaldson/tom/flower_map/data/Week3/6217East',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6217East_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/2017_6217East"')
# upload_jsons('/mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20176217_east',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/2017_6217East"')
# parser = argparse.ArgumentParser(
# description="find images and upload to google drive"
# )
# parser.add_argument(
# "image_range", help="path to the csv file defining the range of images to be uploaded. example seen at /mnt/biology/donaldson/tom/flower_map_new/2017_6217East.csv."
# )
# parser.add_argument(
# "image_dir", help="path to the directory of images to be selected from. example seen at /mnt/biology/donaldson/tom/flower_map/data/Week3/6217East."
# )
# parser.add_argument(
# "output_txt_path", help="path to the file you want to include all the names of the files you are uploading. example seen at '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6217East_summary.txt'."
# )
# parser.add_argument(
# "jsons_path", help="path to the json files to be uploaded. example seen at: /mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20176217_east."
# )
# parser.add_argument(
# "gdrive_path", help="path to upload the files to."
# )
# args = parser.parse_args()
csv_summary_path = input("path to the csv file defining the range of images to be uploaded. example seen at /mnt/biology/donaldson/tom/flower_map_new/2017_6217East.csv")
image_dir = input("path to the directory of images to be selected from. example seen at /mnt/biology/donaldson/tom/flower_map/data/Week3/6217East")
output_txt_path = input("path to the file you want to include all the names of the files you are uploading. example seen at /mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6217East_summary.txt")
jsons_path = input("path to the json files to be uploaded. example seen at: /mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20176217_east")
gdrive_path = input('path to upload the files to, example: "knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/2017_6217East", note that the quote marks are necessary')
def find_images(csv_summary_path, directory_path, output_txt_path, gDrive_path):
a = pd.read_csv(csv_summary_path)
start_list = a['image_name_start'].tolist()
end_list = a['image_name_end'].tolist()
all_contained_image_names = []
print("prep start, identifying all images")
for index in range(len(start_list)):
current_start = start_list[index]
current_end = end_list[index]
current_prefix = ""
current_prefix_list = current_start.split("_")[:-1]
for el in current_prefix_list:
current_prefix += el + "_"
current_start_number = int(current_start.split("_")[-1])
current_end_number = int(current_end.split("_")[-1])
current_range = range(current_start_number, current_end_number)
current_range_image_names = []
for image in current_range:
if len(str(image)) == 2:
current_range_image_names.append(current_prefix+'00'+str(image)+".JPG")
elif len(str(image)) == 3:
current_range_image_names.append(current_prefix+'0'+str(image)+".JPG")
elif len(str(image)) == 1:
current_range_image_names.append(current_prefix+'000'+str(image)+".JPG")
all_contained_image_names.extend(current_range_image_names)
print("writing image names to txt file at", output_txt_path)
with open(output_txt_path, 'w') as f:
for item in all_contained_image_names:
f.write("%s\n" % item)
include_from = '--include-from='+output_txt_path
process = 'rclone copy '+directory_path+" "+gDrive_path+" "+include_from +" -v"
print("prep finished, run the following command to upload all chosen images")
print(process)
return all_contained_image_names
# find_images('/mnt/biology/donaldson/tom/flower_map_new/070921_NorthHasPlants.csv',
# '/mnt/biology/donaldson/tom/flower_map_new/newData/070921_North',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/0709_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/dataToBeLabelled/070921_North"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/071121_CentralEasternHasPlants.csv',
# '/mnt/biology/donaldson/tom/flower_map_new/newData/071121_CentralEastern',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/0711_CentralEastern_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/dataToBeLabelled/071121_CentralEastern"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/071621_South.csv',
# '/mnt/biology/donaldson/tom/flower_map_new/newData/071621_South',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/071621_South_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/dataToBeLabelled/071621_South"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/071121_Western.csv',
# '/mnt/biology/donaldson/tom/flower_map_new/newData/071121_Western',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/071121_Western_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/dataToBeLabelled/071121_Western"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/2017_6217East.csv',
# '/mnt/biology/donaldson/tom/flower_map/data/Week3/6217East',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6217East_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/dataToBeLabelled/2017_6217East"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/2017_6617East1.csv',
# '/mnt/biology/donaldson/tom/flower_map/data/Week4/6617East1',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6617East1_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/dataToBeLabelled/2017_6617East1"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/2017_6617East2.csv',
# '/mnt/biology/donaldson/tom/flower_map/data/Week4/6617East2',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6617East2_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/dataToBeLabelled/2017_6617East2"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/2017_6917West.csv',
# '/mnt/biology/donaldson/tom/flower_map/data/Week4/6917West',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6917West_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/dataToBeLabelled/2017_6917West"')
def upload_jsons(json_folder, gDrive_path):
process = 'rclone copy '+json_folder+" "+gDrive_path+" "+" -v"
print("prep finished, run the following command to upload all chosen images")
print(process)
# process = Popen(['rclone', 'copy', directory_path, gDrive_path, include_from, '-v'], stdout=PIPE, stderr=PIPE)
# stdout, stderr = process.communicate()
return
find_images(csv_summary_path, image_dir, output_txt_path, gdrive_path)
upload_jsons(jsons_path, gdrive_path)
# upload_jsons('/mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20176217_east',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/dataToBeLabelled/2017_6217East_json"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/2017_6217East.csv',
# '/mnt/biology/donaldson/tom/flower_map/data/Week3/6217East',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6217East_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/dataToBeLabelled/2017_6217East_json"')
# upload_jsons('/mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20176917_west',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/dataToBeLabelled/2017_6917West_json"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/2017_6917West.csv',
# '/mnt/biology/donaldson/tom/flower_map/data/Week4/6917West',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6917West_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/dataToBeLabelled/2017_6917West_json"')
#### Final Data Uploads
# find_images('/mnt/biology/donaldson/tom/flower_map_new/070921_NorthHasPlants.csv',
# '/mnt/biology/donaldson/tom/flower_map_new/newData/070921_North',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/0709_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/070921_North"')
# upload_jsons('/mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20210709_north',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/070921_North"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/071121_CentralEasternHasPlants.csv',
# '/mnt/biology/donaldson/tom/flower_map_new/newData/071121_CentralEastern',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/0711_CentralEastern_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/071121_CentralEastern"')
# upload_jsons('/mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20210711_centralEastern',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/071121_CentralEastern"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/071621_South.csv',
# '/mnt/biology/donaldson/tom/flower_map_new/newData/071621_South',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/071621_South_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/071621_South"')
# upload_jsons('/mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20210716_south',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/071621_South"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/071121_Western.csv',
# '/mnt/biology/donaldson/tom/flower_map_new/newData/071121_Western',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/071121_Western_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/071121_Western"')
# upload_jsons('/mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20210711_western',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/071121_Western"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/2017_6217East.csv',
# '/mnt/biology/donaldson/tom/flower_map/data/Week3/6217East',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6217East_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/2017_6217East"')
# upload_jsons('/mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20176217_east',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/2017_6217East"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/2017_6617East1.csv',
# '/mnt/biology/donaldson/tom/flower_map/data/Week4/6617East1',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6617East1_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/2017_6617East1"')
# upload_jsons('/mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20176617_east1',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/2017_6617East1"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/2017_6617East2.csv',
# '/mnt/biology/donaldson/tom/flower_map/data/Week4/6617East2',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6617East2_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/2017_6617East2"')
# upload_jsons('/mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20176617_east2',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/2017_6617East2"')
# find_images('/mnt/biology/donaldson/tom/flower_map_new/2017_6917West.csv',
# '/mnt/biology/donaldson/tom/flower_map/data/Week4/6917West',
# '/mnt/biology/donaldson/tom/flower_map_new/useful_images_summaries/2017_6917West_summary.txt',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/2017_6917West"')
# upload_jsons('/mnt/biology/donaldson/tom/flower_map_new/annotations/jsons_20176917_west',
# '"knuthXGDrive:/Bee Lab/Projects/Bee Forage Mapping/Bee Forage Mapping - Tom Thesis/labelled data/2017_6917West"')
# if __name__ == "__main__":
# find_images(args.image_range, args.image_dir, args.output_txt_path, '\"'+args.gdrive_path+'\"')
# upload_jsons(args.jsons_path, '\"'+args.gdrive_path+'\"')