From ee0868dcfcaab39914a57d61f5dee18904d4e6d3 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Thu, 8 Aug 2024 11:36:45 -0700 Subject: [PATCH 01/35] init. Need to test on non-musl system --- lib/c/compile.sh | 4 +++ lib/c/example.fastq | 16 +++++++++++ lib/c/lib.go | 70 +++++++++++++++++++++++++++++++++++++++++++++ lib/c/test.py | 57 ++++++++++++++++++++++++++++++++++++ lib/c/testlib.c | 9 ++++++ 5 files changed, 156 insertions(+) create mode 100755 lib/c/compile.sh create mode 100644 lib/c/example.fastq create mode 100644 lib/c/lib.go create mode 100644 lib/c/test.py create mode 100644 lib/c/testlib.c diff --git a/lib/c/compile.sh b/lib/c/compile.sh new file mode 100755 index 00000000..160479e1 --- /dev/null +++ b/lib/c/compile.sh @@ -0,0 +1,4 @@ +go build -o libawesome.so -buildmode=c-shared lib.go +gcc testlib.c -o testlib -L. -lawesome -Wl,-rpath='$ORIGIN' +./testlib +rm testlib diff --git a/lib/c/example.fastq b/lib/c/example.fastq new file mode 100644 index 00000000..c1d451c9 --- /dev/null +++ b/lib/c/example.fastq @@ -0,0 +1,16 @@ +@e3cc70d5-90ef-49b6-bbe1-cfef99537d73 runid=99790f25859e24307203c25273f3a8be8283e7eb read=13956 ch=53 start_time=2020-11-11T01:49:01Z flow_cell_id=AEI083 protocol_group_id=NanoSav2 sample_id=nanosavseq2 +GATGTGCGCCGTTCCAGTTGCGACGTACTATAATCCCCGGCAACACGGTGCTGATTCTCTTCCTGTTCCAGAAAGCATAAACAGATGCAAGTCTGGTGTGATTAACTTCACCAAAGGGCTGGTTGTAATATTAGGAAATCTAACAATAGATTCTGTTGGTTGGACTCTAAAATTAGAAATTTGATAGATTCCTTTTCCCAAATGAAAGTTTAACGTACACTTTGTTTCTAAAGGAAGGTCAAATTACAGTCTACAGCATCGTAATGGTTCATTTTCATTTATATTTTAATACTAGAAAAGTCCTAGGTTGAAGATAACCACATAATAAGCTGCAACTTCAGCTGTCCCAACCTGAAGAAGAATCGCAGGAGTCGAAATAACTTCTGTAAAGCAAGTAGTTTGAACCTATTGATGTTTCAACATGAGCAATACGTAACT ++ +$$&%&%#$)*59;/767C378411,***,('11<;:,0039/0&()&'2(/*((4.1.09751).601+'#&&&,-**/0-+3558,/)+&)'&&%&$$'%'%'&*/5978<9;**'3*'&&A?99:;:97:278?=9B?CLJHGG=9<@AC@@=>?=>D>=3<>=>3362$%/((+/%&+//.-,%-4:+..000,&$#%$$%+*)&*0%.//*?<<;>DE>.8942&&//074&$033)*&&&%**)%)962133-%'&*99><<=1144??6.027639.011/-)($#$(/422*4;:=122>?@6964:.5'8:52)*675=:4@;323&&##'.-57*4597)+0&:7<7-550REGB21/0+*79/&/6538())+)+23665+(''$$$'-2(&&*-.-#$&%%$$,-)&$$#$'&,);;9'04;:EB91*999211%;9/7<:=(,%%%)7<@9(+--5/679;:9AA.0024139)&'%?AI@;=91374(--6=773;3445))1*/459:EDB>=A4446+))&&$&'"##%##)+&-/3';;.685&8>24+'#&+++78:;9643*)&&'19+$%$,%%#')8<?LQKGBFGHCHD>AA=664741&)*%%.0'-'%&&%$(55:7=9@B@==A>:<7*$&,,14>@<,-9>54/-D==9:>?&'&'('),/%&%&&(&#$%''.)&,)*&(.)))%)*5;>7..+*%%$$'479<54*<<:''%($',*8:;?BA/-(,&0769963,,*/644&%)&&&++.++.--,)%%& +@60907b6b-5e38-498e-9c07-f036ebd8c658 runid=99790f25859e24307203c25273f3a8be8283e7eb read=13962 ch=53 start_time=2020-11-11T01:49:07Z flow_cell_id=AEI083 protocol_group_id=NanoSav2 sample_id=nanosavseq2 +AGTGTGCTTCGTTCAGTTGCGTATTGCTACTTACTAACGAGGTCTTCCTTGTATGTTGAGTGAGAGCGGTGAACCAAGACGCAGTATTATTGGGTAAACCTTAGGCCGACGTTGTTTTGATCGCGCCCCACTGCCGTTCTCCATTCTGGTTGCACCAGTTGAATCTGAGGTCCACCAAACGTAATGCGGGGTGCATTTCGCTGATTTGGGGTCCATTATCAGACATTTTAGTTTGTTCGTTTAGATGAAATCTAAAAACAACACGAACGTCATGATACTCTAAAAAGTCTTCATAGAACAGACAACGCTACAGACTACCCAATTTGGGTTCCTGGCAATTAATTGTAAAAAAGGTAAGCGAGCAACCTCATTCTCGATATCGATGTACTGAATACGTTGATTTAGAACCAGCCTCAACTCCGTCTTGCGACAATATGCGCTT ++ +%')'++124>AB,93;36-,,-//3695.00046+,)*)%%324>B;../%,&..85(''/-?;:8:7;AEFIJE3=9:7942:><11&*+969796+-0&'&))+-2952;?:HLC86:;:-%%%%%%+'%$')67:1160()'$&$'&0$#$#%'*7>BCA?>01&51&&5$$D?AG?7<3))';7-**@?BBD)8:-(%111/.027=?ADD13=JLAE==?B6$((+77;+''-%&&&.79;-8/.87<;794---,6558622%#$%(67,37*%+**+ +@990e110e-5e50-41a2-8ad5-92044d4465b8 runid=99790f25859e24307203c25273f3a8be8283e7eb read=15011 ch=52 start_time=2020-11-11T01:49:18Z flow_cell_id=AEI083 protocol_group_id=NanoSav2 sample_id=nanosavseq2 +GGTATACTTCGTTCAGTTACGTATTGCTCAAGACGGAGTTGAGGCTGGTTCTAAATCACCCATTCAGTACATCGATATCGATAGTGCCACGGTTTCCTGTTTTCGCACACATTGTTACCAGAACCTAAGTGGGTAGTCTTGTAGTGCGTTGTTCGTTTCTATGAAGACTTTTTAGAGTATCATGACGTTCGTGTGCTGAATTTCATCTAAGCGAACAAACTAAAATGTCTGATAATGGACCCCAAAATCAACGAAATGCGCCCCGCGTGCGTTTGTTGGTTCCCTCAGATTCGCTGGCAGTAACCAGAATGGAGAACGCAGTGGGGCGCGATCAAAACAGCATCGGCCCCAAGGTTTGCCAATAATACTGCGTCTTGGTTCCACCGCTCTCACTCAACATGGCAAGGAAGACCTCGTTAGTAGTAGCAATACGTAACC ++ +%&&%$&(()')23-+.:)''.10355=DEAE@E;--&+:<115924-0CJ:<>DE?=6,.+,.//0*123,*7//&'&'%#&$$)*631-/0&%($),&%)(+/0-/29;88=;8EGFHFJFEFFFB===C@?;((426?=<5&':;<8&8()%76:?5.'2-,'()/&20.-3>8+$#'&.1186EBA@B>C;:-/)+...0-+%1-/3*&.)$'(&'$'1&,466663)5+<6)++,.7999;;92;9:977$+61)-124.5970<,8=:-.1--,+'*++(-***,,12@??9:2/61-)&## diff --git a/lib/c/lib.go b/lib/c/lib.go new file mode 100644 index 00000000..023dd240 --- /dev/null +++ b/lib/c/lib.go @@ -0,0 +1,70 @@ +package main + +/* +#include +#include + +typedef struct { + char* identifier; + char* sequence; + char* quality; + char* optionals; // Serialized JSON string of the map. +} FastqRead; +*/ +import "C" +import ( + "encoding/json" + "io" + "unsafe" + + "github.com/koeng101/dnadesign/lib/bio" +) + +// Function to create an io.Reader from a C FILE*. +func readerFromCFile(cfile *C.FILE) io.Reader { + return &fileReader{file: cfile} +} + +type fileReader struct { + file *C.FILE +} + +func (f *fileReader) Read(p []byte) (n int, err error) { + buffer := (*C.char)(unsafe.Pointer(&p[0])) + count := C.size_t(len(p)) + result := C.fread(unsafe.Pointer(buffer), 1, count, f.file) + if result == 0 { + if C.feof(f.file) != 0 { + return 0, io.EOF + } + return 0, io.ErrUnexpectedEOF + } + return int(result), nil +} + +//export ParseFastqFromCFile +func ParseFastqFromCFile(cfile *C.FILE) (*C.FastqRead, int, *C.char) { + reader := readerFromCFile(cfile) + parser := bio.NewFastqParser(reader) + reads, err := parser.Parse() + if err != nil { + return nil, 0, C.CString(err.Error()) + } + + cReads := (*C.FastqRead)(C.malloc(C.size_t(len(reads)) * C.size_t(unsafe.Sizeof(C.FastqRead{})))) + slice := (*[1<<30 - 1]C.FastqRead)(unsafe.Pointer(cReads))[:len(reads):len(reads)] + + for i, read := range reads { + optionalsJSON, _ := json.Marshal(read.Optionals) + slice[i].identifier = C.CString(read.Identifier) + slice[i].sequence = C.CString(read.Sequence) + slice[i].quality = C.CString(read.Quality) + slice[i].optionals = C.CString(string(optionalsJSON)) + } + + return cReads, int(len(reads)), nil +} + +func main() { + // Optionally add any test code or leave empty. +} diff --git a/lib/c/test.py b/lib/c/test.py new file mode 100644 index 00000000..8811dc7c --- /dev/null +++ b/lib/c/test.py @@ -0,0 +1,57 @@ +from __future__ import print_function +import sys +import os +from cffi import FFI + +ffi = FFI() + +# Define common types based on the platform architecture +is_64b = sys.maxsize > 2**32 +if is_64b: + ffi.cdef("typedef long GoInt;\n") +else: + ffi.cdef("typedef int GoInt;\n") + +# Define the FastqRead struct and the function signature for ParseFastqFromCFile +ffi.cdef(""" +typedef struct { + char* identifier; + char* sequence; + char* quality; + char* optionals; // Serialized JSON string of the map. +} FastqRead; + +typedef struct { + FastqRead* reads; + GoInt numReads; + char* error; +} FastqResult; + +FastqResult ParseFastqFromCFile(void* cfile); +""") + + +# Load the shared library compiled from Go +lib = ffi.dlopen("./libawesome.so") + +# Use ffi to open a file that can be passed to the Go function +cfile = ffi.cast("void*", ffi.dlopen("./example.fastq", ffi.RTLD_LAZY)) + +# Call the function from the shared library +result = lib.ParseFastqFromCFile(cfile) + +# Check if there was an error +if result.error != ffi.NULL: + error_str = ffi.string(result.error).decode('utf-8') + print("Error parsing FASTQ:", error_str) +else: + # Process the reads + num_reads = result.numReads + reads = ffi.cast("FastqRead*", result.reads) + for i in range(num_reads): + identifier = ffi.string(reads[i].identifier).decode('utf-8') + sequence = ffi.string(reads[i].sequence).decode('utf-8') + quality = ffi.string(reads[i].quality).decode('utf-8') + optionals = ffi.string(reads[i].optionals).decode('utf-8') + print(f"Read {i+1}: Identifier={identifier}, Sequence={sequence}, Quality={quality}, Optionals={optionals}") + diff --git a/lib/c/testlib.c b/lib/c/testlib.c new file mode 100644 index 00000000..33684f6e --- /dev/null +++ b/lib/c/testlib.c @@ -0,0 +1,9 @@ +// testlib.c +#include + +#include "libawesome.h" // Ensure this header file is correct. + +int main() { + printf("Library loaded successfully.\n"); + return 0; +} From 8408bdeb2a86220b78004d353b8bc67ad691798a Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Thu, 8 Aug 2024 18:54:19 +0000 Subject: [PATCH 02/35] functioning. --- lib/c/compile.sh | 1 + lib/c/test.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/c/compile.sh b/lib/c/compile.sh index 160479e1..4683c4cb 100755 --- a/lib/c/compile.sh +++ b/lib/c/compile.sh @@ -2,3 +2,4 @@ go build -o libawesome.so -buildmode=c-shared lib.go gcc testlib.c -o testlib -L. -lawesome -Wl,-rpath='$ORIGIN' ./testlib rm testlib +python3 test.py diff --git a/lib/c/test.py b/lib/c/test.py index 8811dc7c..8c3b4416 100644 --- a/lib/c/test.py +++ b/lib/c/test.py @@ -14,6 +14,11 @@ # Define the FastqRead struct and the function signature for ParseFastqFromCFile ffi.cdef(""" +typedef struct FILE FILE; +FILE *fopen(const char *path, const char *mode); +int fclose(FILE *fp); + + typedef struct { char* identifier; char* sequence; @@ -34,8 +39,9 @@ # Load the shared library compiled from Go lib = ffi.dlopen("./libawesome.so") -# Use ffi to open a file that can be passed to the Go function -cfile = ffi.cast("void*", ffi.dlopen("./example.fastq", ffi.RTLD_LAZY)) +file_path = "example.fastq".encode('utf-8') # Convert the string to bytes +mode = "r".encode('utf-8') # Convert the mode to bytes as well +cfile = lib.fopen(file_path, mode) # Call the function from the shared library result = lib.ParseFastqFromCFile(cfile) From 0eb57e3c44d8178658b0095dce4aae5879b9ffd8 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Thu, 8 Aug 2024 18:55:22 +0000 Subject: [PATCH 03/35] tests now pass --- lib/c/testlib.c | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 lib/c/testlib.c diff --git a/lib/c/testlib.c b/lib/c/testlib.c deleted file mode 100644 index 33684f6e..00000000 --- a/lib/c/testlib.c +++ /dev/null @@ -1,9 +0,0 @@ -// testlib.c -#include - -#include "libawesome.h" // Ensure this header file is correct. - -int main() { - printf("Library loaded successfully.\n"); - return 0; -} From bbe1810581820376f20259898b14bf3596c35460 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Thu, 8 Aug 2024 18:55:48 +0000 Subject: [PATCH 04/35] update script to remove c intermediate --- lib/c/compile.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/c/compile.sh b/lib/c/compile.sh index 4683c4cb..225da0ed 100755 --- a/lib/c/compile.sh +++ b/lib/c/compile.sh @@ -1,5 +1,2 @@ go build -o libawesome.so -buildmode=c-shared lib.go -gcc testlib.c -o testlib -L. -lawesome -Wl,-rpath='$ORIGIN' -./testlib -rm testlib python3 test.py From a673302fb25fc8a017c1694307aa42a5f75a6501 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Thu, 8 Aug 2024 11:58:14 -0700 Subject: [PATCH 05/35] linter happy --- lib/c/lib.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/c/lib.go b/lib/c/lib.go index 023dd240..a726d357 100644 --- a/lib/c/lib.go +++ b/lib/c/lib.go @@ -62,7 +62,7 @@ func ParseFastqFromCFile(cfile *C.FILE) (*C.FastqRead, int, *C.char) { slice[i].optionals = C.CString(string(optionalsJSON)) } - return cReads, int(len(reads)), nil + return cReads, len(reads), nil } func main() { From 19fe597e4b9f4519ae6dcad2ae1d9042d3d31759 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Thu, 8 Aug 2024 19:27:10 +0000 Subject: [PATCH 06/35] split into definitions header --- lib/c/definitions.h | 19 +++++++++++++++++++ lib/c/test.py | 26 +++----------------------- 2 files changed, 22 insertions(+), 23 deletions(-) create mode 100644 lib/c/definitions.h diff --git a/lib/c/definitions.h b/lib/c/definitions.h new file mode 100644 index 00000000..868e64f3 --- /dev/null +++ b/lib/c/definitions.h @@ -0,0 +1,19 @@ +typedef struct FILE FILE; +FILE *fopen(const char *path, const char *mode); +int fclose(FILE *fp); + +typedef struct { + char* identifier; + char* sequence; + char* quality; + char* optionals; // Serialized JSON string of the map. +} FastqRead; + +typedef struct { + FastqRead* reads; + GoInt numReads; + char* error; +} FastqResult; + +FastqResult ParseFastqFromCFile(void* cfile); + diff --git a/lib/c/test.py b/lib/c/test.py index 8c3b4416..1ec64190 100644 --- a/lib/c/test.py +++ b/lib/c/test.py @@ -12,29 +12,9 @@ else: ffi.cdef("typedef int GoInt;\n") -# Define the FastqRead struct and the function signature for ParseFastqFromCFile -ffi.cdef(""" -typedef struct FILE FILE; -FILE *fopen(const char *path, const char *mode); -int fclose(FILE *fp); - - -typedef struct { - char* identifier; - char* sequence; - char* quality; - char* optionals; // Serialized JSON string of the map. -} FastqRead; - -typedef struct { - FastqRead* reads; - GoInt numReads; - char* error; -} FastqResult; - -FastqResult ParseFastqFromCFile(void* cfile); -""") - +# Read the C declarations from the external file +with open('definitions.h', 'r') as f: + ffi.cdef(f.read()) # Load the shared library compiled from Go lib = ffi.dlopen("./libawesome.so") From daed1bf255e0218ec84434c7bc7dee6243cd2eec Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sat, 10 Aug 2024 20:18:56 +0000 Subject: [PATCH 07/35] made py directory --- go.work | 3 +- lib/c/compile.sh | 2 - lib/c/definitions.h | 19 ----- lib/c/example.fastq | 16 ---- lib/c/lib.go | 70 ------------------ lib/c/test.py | 43 ----------- py/.gitignore | 117 ++++++++++++++++++++++++++++++ py/Makefile | 30 ++++++++ py/README.md | 6 ++ py/dnadesign/__init__.py | 0 py/dnadesign/cffi_bindings.py | 25 +++++++ py/dnadesign/data/example.fasta | 11 +++ py/dnadesign/definitions.h | 17 +++++ py/dnadesign/fasta_parser.py | 26 +++++++ py/dnadesign/test_fasta_parser.py | 17 +++++ py/go.mod | 7 ++ py/go.sum | 6 ++ py/lib.go | 95 ++++++++++++++++++++++++ py/setup.py | 26 +++++++ 19 files changed, 385 insertions(+), 151 deletions(-) delete mode 100755 lib/c/compile.sh delete mode 100644 lib/c/definitions.h delete mode 100644 lib/c/example.fastq delete mode 100644 lib/c/lib.go delete mode 100644 lib/c/test.py create mode 100644 py/.gitignore create mode 100644 py/Makefile create mode 100644 py/README.md create mode 100644 py/dnadesign/__init__.py create mode 100644 py/dnadesign/cffi_bindings.py create mode 100644 py/dnadesign/data/example.fasta create mode 100644 py/dnadesign/definitions.h create mode 100644 py/dnadesign/fasta_parser.py create mode 100644 py/dnadesign/test_fasta_parser.py create mode 100644 py/go.mod create mode 100644 py/go.sum create mode 100644 py/lib.go create mode 100644 py/setup.py diff --git a/go.work b/go.work index b7479224..1c851ee7 100644 --- a/go.work +++ b/go.work @@ -1,6 +1,7 @@ -go 1.22.0 +go 1.22.5 use ( ./external ./lib + ./py ) diff --git a/lib/c/compile.sh b/lib/c/compile.sh deleted file mode 100755 index 225da0ed..00000000 --- a/lib/c/compile.sh +++ /dev/null @@ -1,2 +0,0 @@ -go build -o libawesome.so -buildmode=c-shared lib.go -python3 test.py diff --git a/lib/c/definitions.h b/lib/c/definitions.h deleted file mode 100644 index 868e64f3..00000000 --- a/lib/c/definitions.h +++ /dev/null @@ -1,19 +0,0 @@ -typedef struct FILE FILE; -FILE *fopen(const char *path, const char *mode); -int fclose(FILE *fp); - -typedef struct { - char* identifier; - char* sequence; - char* quality; - char* optionals; // Serialized JSON string of the map. -} FastqRead; - -typedef struct { - FastqRead* reads; - GoInt numReads; - char* error; -} FastqResult; - -FastqResult ParseFastqFromCFile(void* cfile); - diff --git a/lib/c/example.fastq b/lib/c/example.fastq deleted file mode 100644 index c1d451c9..00000000 --- a/lib/c/example.fastq +++ /dev/null @@ -1,16 +0,0 @@ -@e3cc70d5-90ef-49b6-bbe1-cfef99537d73 runid=99790f25859e24307203c25273f3a8be8283e7eb read=13956 ch=53 start_time=2020-11-11T01:49:01Z flow_cell_id=AEI083 protocol_group_id=NanoSav2 sample_id=nanosavseq2 -GATGTGCGCCGTTCCAGTTGCGACGTACTATAATCCCCGGCAACACGGTGCTGATTCTCTTCCTGTTCCAGAAAGCATAAACAGATGCAAGTCTGGTGTGATTAACTTCACCAAAGGGCTGGTTGTAATATTAGGAAATCTAACAATAGATTCTGTTGGTTGGACTCTAAAATTAGAAATTTGATAGATTCCTTTTCCCAAATGAAAGTTTAACGTACACTTTGTTTCTAAAGGAAGGTCAAATTACAGTCTACAGCATCGTAATGGTTCATTTTCATTTATATTTTAATACTAGAAAAGTCCTAGGTTGAAGATAACCACATAATAAGCTGCAACTTCAGCTGTCCCAACCTGAAGAAGAATCGCAGGAGTCGAAATAACTTCTGTAAAGCAAGTAGTTTGAACCTATTGATGTTTCAACATGAGCAATACGTAACT -+ -$$&%&%#$)*59;/767C378411,***,('11<;:,0039/0&()&'2(/*((4.1.09751).601+'#&&&,-**/0-+3558,/)+&)'&&%&$$'%'%'&*/5978<9;**'3*'&&A?99:;:97:278?=9B?CLJHGG=9<@AC@@=>?=>D>=3<>=>3362$%/((+/%&+//.-,%-4:+..000,&$#%$$%+*)&*0%.//*?<<;>DE>.8942&&//074&$033)*&&&%**)%)962133-%'&*99><<=1144??6.027639.011/-)($#$(/422*4;:=122>?@6964:.5'8:52)*675=:4@;323&&##'.-57*4597)+0&:7<7-550REGB21/0+*79/&/6538())+)+23665+(''$$$'-2(&&*-.-#$&%%$$,-)&$$#$'&,);;9'04;:EB91*999211%;9/7<:=(,%%%)7<@9(+--5/679;:9AA.0024139)&'%?AI@;=91374(--6=773;3445))1*/459:EDB>=A4446+))&&$&'"##%##)+&-/3';;.685&8>24+'#&+++78:;9643*)&&'19+$%$,%%#')8<?LQKGBFGHCHD>AA=664741&)*%%.0'-'%&&%$(55:7=9@B@==A>:<7*$&,,14>@<,-9>54/-D==9:>?&'&'('),/%&%&&(&#$%''.)&,)*&(.)))%)*5;>7..+*%%$$'479<54*<<:''%($',*8:;?BA/-(,&0769963,,*/644&%)&&&++.++.--,)%%& -@60907b6b-5e38-498e-9c07-f036ebd8c658 runid=99790f25859e24307203c25273f3a8be8283e7eb read=13962 ch=53 start_time=2020-11-11T01:49:07Z flow_cell_id=AEI083 protocol_group_id=NanoSav2 sample_id=nanosavseq2 -AGTGTGCTTCGTTCAGTTGCGTATTGCTACTTACTAACGAGGTCTTCCTTGTATGTTGAGTGAGAGCGGTGAACCAAGACGCAGTATTATTGGGTAAACCTTAGGCCGACGTTGTTTTGATCGCGCCCCACTGCCGTTCTCCATTCTGGTTGCACCAGTTGAATCTGAGGTCCACCAAACGTAATGCGGGGTGCATTTCGCTGATTTGGGGTCCATTATCAGACATTTTAGTTTGTTCGTTTAGATGAAATCTAAAAACAACACGAACGTCATGATACTCTAAAAAGTCTTCATAGAACAGACAACGCTACAGACTACCCAATTTGGGTTCCTGGCAATTAATTGTAAAAAAGGTAAGCGAGCAACCTCATTCTCGATATCGATGTACTGAATACGTTGATTTAGAACCAGCCTCAACTCCGTCTTGCGACAATATGCGCTT -+ -%')'++124>AB,93;36-,,-//3695.00046+,)*)%%324>B;../%,&..85(''/-?;:8:7;AEFIJE3=9:7942:><11&*+969796+-0&'&))+-2952;?:HLC86:;:-%%%%%%+'%$')67:1160()'$&$'&0$#$#%'*7>BCA?>01&51&&5$$D?AG?7<3))';7-**@?BBD)8:-(%111/.027=?ADD13=JLAE==?B6$((+77;+''-%&&&.79;-8/.87<;794---,6558622%#$%(67,37*%+**+ -@990e110e-5e50-41a2-8ad5-92044d4465b8 runid=99790f25859e24307203c25273f3a8be8283e7eb read=15011 ch=52 start_time=2020-11-11T01:49:18Z flow_cell_id=AEI083 protocol_group_id=NanoSav2 sample_id=nanosavseq2 -GGTATACTTCGTTCAGTTACGTATTGCTCAAGACGGAGTTGAGGCTGGTTCTAAATCACCCATTCAGTACATCGATATCGATAGTGCCACGGTTTCCTGTTTTCGCACACATTGTTACCAGAACCTAAGTGGGTAGTCTTGTAGTGCGTTGTTCGTTTCTATGAAGACTTTTTAGAGTATCATGACGTTCGTGTGCTGAATTTCATCTAAGCGAACAAACTAAAATGTCTGATAATGGACCCCAAAATCAACGAAATGCGCCCCGCGTGCGTTTGTTGGTTCCCTCAGATTCGCTGGCAGTAACCAGAATGGAGAACGCAGTGGGGCGCGATCAAAACAGCATCGGCCCCAAGGTTTGCCAATAATACTGCGTCTTGGTTCCACCGCTCTCACTCAACATGGCAAGGAAGACCTCGTTAGTAGTAGCAATACGTAACC -+ -%&&%$&(()')23-+.:)''.10355=DEAE@E;--&+:<115924-0CJ:<>DE?=6,.+,.//0*123,*7//&'&'%#&$$)*631-/0&%($),&%)(+/0-/29;88=;8EGFHFJFEFFFB===C@?;((426?=<5&':;<8&8()%76:?5.'2-,'()/&20.-3>8+$#'&.1186EBA@B>C;:-/)+...0-+%1-/3*&.)$'(&'$'1&,466663)5+<6)++,.7999;;92;9:977$+61)-124.5970<,8=:-.1--,+'*++(-***,,12@??9:2/61-)&## diff --git a/lib/c/lib.go b/lib/c/lib.go deleted file mode 100644 index a726d357..00000000 --- a/lib/c/lib.go +++ /dev/null @@ -1,70 +0,0 @@ -package main - -/* -#include -#include - -typedef struct { - char* identifier; - char* sequence; - char* quality; - char* optionals; // Serialized JSON string of the map. -} FastqRead; -*/ -import "C" -import ( - "encoding/json" - "io" - "unsafe" - - "github.com/koeng101/dnadesign/lib/bio" -) - -// Function to create an io.Reader from a C FILE*. -func readerFromCFile(cfile *C.FILE) io.Reader { - return &fileReader{file: cfile} -} - -type fileReader struct { - file *C.FILE -} - -func (f *fileReader) Read(p []byte) (n int, err error) { - buffer := (*C.char)(unsafe.Pointer(&p[0])) - count := C.size_t(len(p)) - result := C.fread(unsafe.Pointer(buffer), 1, count, f.file) - if result == 0 { - if C.feof(f.file) != 0 { - return 0, io.EOF - } - return 0, io.ErrUnexpectedEOF - } - return int(result), nil -} - -//export ParseFastqFromCFile -func ParseFastqFromCFile(cfile *C.FILE) (*C.FastqRead, int, *C.char) { - reader := readerFromCFile(cfile) - parser := bio.NewFastqParser(reader) - reads, err := parser.Parse() - if err != nil { - return nil, 0, C.CString(err.Error()) - } - - cReads := (*C.FastqRead)(C.malloc(C.size_t(len(reads)) * C.size_t(unsafe.Sizeof(C.FastqRead{})))) - slice := (*[1<<30 - 1]C.FastqRead)(unsafe.Pointer(cReads))[:len(reads):len(reads)] - - for i, read := range reads { - optionalsJSON, _ := json.Marshal(read.Optionals) - slice[i].identifier = C.CString(read.Identifier) - slice[i].sequence = C.CString(read.Sequence) - slice[i].quality = C.CString(read.Quality) - slice[i].optionals = C.CString(string(optionalsJSON)) - } - - return cReads, len(reads), nil -} - -func main() { - // Optionally add any test code or leave empty. -} diff --git a/lib/c/test.py b/lib/c/test.py deleted file mode 100644 index 1ec64190..00000000 --- a/lib/c/test.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import print_function -import sys -import os -from cffi import FFI - -ffi = FFI() - -# Define common types based on the platform architecture -is_64b = sys.maxsize > 2**32 -if is_64b: - ffi.cdef("typedef long GoInt;\n") -else: - ffi.cdef("typedef int GoInt;\n") - -# Read the C declarations from the external file -with open('definitions.h', 'r') as f: - ffi.cdef(f.read()) - -# Load the shared library compiled from Go -lib = ffi.dlopen("./libawesome.so") - -file_path = "example.fastq".encode('utf-8') # Convert the string to bytes -mode = "r".encode('utf-8') # Convert the mode to bytes as well -cfile = lib.fopen(file_path, mode) - -# Call the function from the shared library -result = lib.ParseFastqFromCFile(cfile) - -# Check if there was an error -if result.error != ffi.NULL: - error_str = ffi.string(result.error).decode('utf-8') - print("Error parsing FASTQ:", error_str) -else: - # Process the reads - num_reads = result.numReads - reads = ffi.cast("FastqRead*", result.reads) - for i in range(num_reads): - identifier = ffi.string(reads[i].identifier).decode('utf-8') - sequence = ffi.string(reads[i].sequence).decode('utf-8') - quality = ffi.string(reads[i].quality).decode('utf-8') - optionals = ffi.string(reads[i].optionals).decode('utf-8') - print(f"Read {i+1}: Identifier={identifier}, Sequence={sequence}, Quality={quality}, Optionals={optionals}") - diff --git a/py/.gitignore b/py/.gitignore new file mode 100644 index 00000000..0489c125 --- /dev/null +++ b/py/.gitignore @@ -0,0 +1,117 @@ +dnadesign/lib +dnadesign/libdnadesign.h +venv + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypi.org project descriptions, pipenv is a dependable Python packaging tool. +# it generates the following which should not be included in your version control system. +Pipfile.lock + +# poetry +poetry.lock + +# MyPy +.mypy_cache/ +dmypy.json +dmypy.json.# + +# Pyre type checker +.pyre/ + +# PyCharm + all JetBrains IDEs +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, GoLand +.idea/ +*.iml + +# VSCode +.vscode/ + +# Environment variables +.env +.envrc + diff --git a/py/Makefile b/py/Makefile new file mode 100644 index 00000000..9a90561c --- /dev/null +++ b/py/Makefile @@ -0,0 +1,30 @@ +.PHONY: all clean build + +# List of architectures and operating systems to support +ARCHS = amd64 arm64 +OSES = linux darwin windows + +# Mapping Go architectures to Zig architectures +ZIG_ARCHS_amd64 = x86_64 +ZIG_ARCHS_arm64 = aarch64 + +# Mapping Go operating systems to Zig targets +ZIG_OSES_linux = linux-gnu +ZIG_OSES_darwin = macos +ZIG_OSES_windows = windows-gnu + +# Default target +all: build + +# Build libraries for all architecture and OS combinations +build: + $(foreach os,$(OSES),\ + $(foreach arch,$(ARCHS),\ + CC="zig cc -target $(ZIG_ARCHS_$(arch))-$(ZIG_OSES_$(os))" \ + CGO_ENABLED=1 GOOS=$(os) GOARCH=$(arch) go build -o dnadesign/lib/libdnadesign_$(os)_$(arch).so -buildmode=c-shared lib.go;\ + )\ + ) + +clean: + rm -f dnadesign/lib/libdnadesign_*.so + diff --git a/py/README.md b/py/README.md new file mode 100644 index 00000000..00e05e70 --- /dev/null +++ b/py/README.md @@ -0,0 +1,6 @@ +To deploy your package: +1. Run `make` to build the shared libraries for all supported platforms. +2. Run `python3 setup.py sdist bdist_wheel` to create the distribution files. +3. Use `twine` to upload the distribution files to PyPI. + +Simple compilation: `go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go` diff --git a/py/dnadesign/__init__.py b/py/dnadesign/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/py/dnadesign/cffi_bindings.py b/py/dnadesign/cffi_bindings.py new file mode 100644 index 00000000..17f63914 --- /dev/null +++ b/py/dnadesign/cffi_bindings.py @@ -0,0 +1,25 @@ +from cffi import FFI +import os +import sys + +ffi = FFI() + +# Define common types based on the platform architecture +is_64b = sys.maxsize > 2**32 +if is_64b: + ffi.cdef("typedef long GoInt;\n") +else: + ffi.cdef("typedef int GoInt;\n") + +current_dir = os.path.dirname(__file__) + +# Build the path to definitions.h and libdnadesign relative to the current script +definitions_path = os.path.join(current_dir, 'definitions.h') +lib_path = os.path.join(current_dir, 'libdnadesign.so') + +# Read the C declarations from an external file +with open(definitions_path, 'r') as f: + ffi.cdef(f.read()) + +# Load the shared library +lib = ffi.dlopen(lib_path) diff --git a/py/dnadesign/data/example.fasta b/py/dnadesign/data/example.fasta new file mode 100644 index 00000000..94b1ea48 --- /dev/null +++ b/py/dnadesign/data/example.fasta @@ -0,0 +1,11 @@ +>gi|5524211|gb|AAD44166.1| cytochrome b [Elephas maximus maximus] +LCLYTHIGRNIYYGSYLYSETWNTGIMLLLITMATAFMGYVLPWGQMSFWGATVITNLFSAIPYIGTNLV +EWIWGGFSVDKATLNRFFAFHFILPFTMVALAGVHLTFLHETGSNNPLGLTSDSDKIPFHPYYTIKDFLG +LLILILLLLLLALLSPDMLGDPDNHMPADPLNTPLHIKPEWYFLFAYAILRSVPNKLGGVLALFLSIVIL +GLMPFLHTSKHRSMMLRPLSQALFWTLTMDLLTLTWIGSQPVEYPYTIIGQMASILYFSIILAFLPIAGX +IENY + +>MCHU - Calmodulin - Human, rabbit, bovine, rat, and chicken +ADQLTEEQIAEFKEAFSLFDKDGDGTITTKELGTVMRSLGQNPTEAELQDMINEVDADGNGTID +FPEFLTMMARKMKDTDSEEEIREAFRVFDKDGNGYISAAELRHVMTNLGEKLTDEEVDEMIREA +DIDGDGQVNYEEFVQMMTAK* diff --git a/py/dnadesign/definitions.h b/py/dnadesign/definitions.h new file mode 100644 index 00000000..f66ca240 --- /dev/null +++ b/py/dnadesign/definitions.h @@ -0,0 +1,17 @@ +typedef struct FILE FILE; +FILE* fopen(const char* path, const char* mode); +int fclose(FILE* fp); + +typedef struct { + char* identifier; + char* sequence; +} FastaRecord; + +typedef struct { + FastaRecord* records; + GoInt numRecords; + char* error; +} FastaResult; + +FastaResult ParseFastaFromCFile(void* cfile); +FastaResult ParseFastaFromCString(char* cstring); diff --git a/py/dnadesign/fasta_parser.py b/py/dnadesign/fasta_parser.py new file mode 100644 index 00000000..3b7cf1f1 --- /dev/null +++ b/py/dnadesign/fasta_parser.py @@ -0,0 +1,26 @@ +from typing import List, Optional +from .cffi_bindings import ffi, lib + +class FastaRecord: + def __init__(self, identifier: str, sequence: str): + self.identifier = identifier + self.sequence = sequence + +def parse_fasta_from_c_file(file_path: str) -> List[FastaRecord]: + cfile = lib.fopen(file_path.encode('utf-8'), "r".encode('utf-8')) + result = lib.ParseFastaFromCFile(cfile) + return _process_result(result) + +def parse_fasta_from_c_string(cstring: str) -> List[FastaRecord]: + result = lib.ParseFastaFromCString(cstring.encode('utf-8')) + return _process_result(result) + +def _process_result(result) -> List[FastaRecord]: + if result.error != ffi.NULL: + error_str = ffi.string(result.error).decode('utf-8') + raise Exception("Error parsing FASTA: " + error_str) + num_records = result.numRecords + records = ffi.cast("FastaRecord*", result.records) + return [FastaRecord(ffi.string(records[i].identifier).decode('utf-8'), + ffi.string(records[i].sequence).decode('utf-8')) + for i in range(num_records)] diff --git a/py/dnadesign/test_fasta_parser.py b/py/dnadesign/test_fasta_parser.py new file mode 100644 index 00000000..eae8f92a --- /dev/null +++ b/py/dnadesign/test_fasta_parser.py @@ -0,0 +1,17 @@ +import pytest +import os +from .fasta_parser import parse_fasta_from_c_file, parse_fasta_from_c_string, FastaRecord + +def test_parse_fasta_from_c_file(): + current_dir = os.path.dirname(__file__) + example_path = os.path.join(current_dir, 'data/example.fasta') + records = parse_fasta_from_c_file(example_path) + assert len(records) > 0 + assert all(isinstance(r, FastaRecord) for r in records) + +def test_parse_fasta_from_c_string(): + fasta_data = ">test\nATCG\n" + records = parse_fasta_from_c_string(fasta_data) + assert len(records) == 1 + assert records[0].identifier == "test" + assert records[0].sequence == "ATCG" diff --git a/py/go.mod b/py/go.mod new file mode 100644 index 00000000..40ffe8ed --- /dev/null +++ b/py/go.mod @@ -0,0 +1,7 @@ +module github.com/koeng101/dnadesign/py + +go 1.22.5 + +require github.com/koeng101/dnadesign/lib v0.0.0-20240531162423-45295e318ef3 + +require golang.org/x/sync v0.5.0 // indirect diff --git a/py/go.sum b/py/go.sum new file mode 100644 index 00000000..59ee4141 --- /dev/null +++ b/py/go.sum @@ -0,0 +1,6 @@ +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/koeng101/dnadesign/lib v0.0.0-20240531162423-45295e318ef3 h1:sFmsnmeffIPhLUBSdhy+9pIaCBHCTddOApZpE3Wvd2I= +github.com/koeng101/dnadesign/lib v0.0.0-20240531162423-45295e318ef3/go.mod h1:sGDJMyNYf4fMqEwwMj2icJ5PpNNc7RjxvctJTM25pLY= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= diff --git a/py/lib.go b/py/lib.go new file mode 100644 index 00000000..1081a2bb --- /dev/null +++ b/py/lib.go @@ -0,0 +1,95 @@ +package main + +/* +#include +#include + +// FastaRecord +typedef struct { + char* identifier; + char* sequence; +} FastaRecord; +*/ +import "C" +import ( + "io" + "strings" + "unsafe" + + "github.com/koeng101/dnadesign/lib/bio" +) + +/****************************************************************************** +Aug 10, 2024 + +Interoperation with CFile + +******************************************************************************/ + +// Function to create an io.Reader from a C FILE*. +func readerFromCFile(cfile *C.FILE) io.Reader { + return &fileReader{file: cfile} +} + +type fileReader struct { + file *C.FILE +} + +func (f *fileReader) Read(p []byte) (n int, err error) { + buffer := (*C.char)(unsafe.Pointer(&p[0])) + count := C.size_t(len(p)) + result := C.fread(unsafe.Pointer(buffer), 1, count, f.file) + if result == 0 { + if C.feof(f.file) != 0 { + return 0, io.EOF + } + return 0, io.ErrUnexpectedEOF + } + return int(result), nil +} + +/****************************************************************************** +Aug 10, 2024 + +Fasta + +******************************************************************************/ + +// goFastaToCFasta converts an io.Reader to a C.FastaResult +func goFastaToCFasta(reader io.Reader) (*C.FastaRecord, int, *C.char) { + parser := bio.NewFastaParser(reader) + records, err := parser.Parse() + if err != nil { + return nil, 0, C.CString(err.Error()) + } + + cRecords := (*C.FastaRecord)(C.malloc(C.size_t(len(records)) * C.size_t(unsafe.Sizeof(C.FastaRecord{})))) + slice := (*[1<<30 - 1]C.FastaRecord)(unsafe.Pointer(cRecords))[:len(records):len(records)] + + for i, read := range records { + slice[i].identifier = C.CString(read.Identifier) + slice[i].sequence = C.CString(read.Sequence) + } + + return cRecords, len(records), nil +} + +//export ParseFastaFromCFile +func ParseFastaFromCFile(cfile *C.FILE) (*C.FastaRecord, int, *C.char) { + reader := readerFromCFile(cfile) + return goFastaToCFasta(reader) +} + +//export ParseFastaFromCString +func ParseFastaFromCString(cstring *C.char) (*C.FastaRecord, int, *C.char) { + reader := strings.NewReader(C.GoString(cstring)) + return goFastaToCFasta(reader) +} + +/****************************************************************************** + +main.go + +******************************************************************************/ + +func main() {} diff --git a/py/setup.py b/py/setup.py new file mode 100644 index 00000000..1850cc28 --- /dev/null +++ b/py/setup.py @@ -0,0 +1,26 @@ +from setuptools import setup, find_packages +from setuptools.command.install import install +import platform +import os + +class CustomInstall(install): + def run(self): + # Install the appropriate shared library based on the user's platform + system = platform.system().lower() + arch = 'amd64' if platform.machine() in ['x86_64', 'AMD64'] else 'arm64' + lib_file = f'dnadesign/lib/libdnadesign_{system}_{arch}.so' + if not os.path.exists(lib_file): + raise FileNotFoundError(f"Could not find required library: {lib_file}") + os.rename(lib_file, 'dnadesign/libdnadesign.so') + install.run(self) + +setup( + name='dnadesign', + version='0.1.0', + packages=find_packages(), + cmdclass={'install': CustomInstall}, + package_data={'dnadesign': ['libdnadesign_*.so']}, + include_package_data=True, + zip_safe=False +) + From a5234a6685be96a5ee1f39b4885c793e54efeb48 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 17:04:48 +0000 Subject: [PATCH 08/35] Add build --- .github/workflows/build.yml | 83 ++++++++++++++++++++ py/Makefile | 30 ------- py/README.md | 1 + py/setup.py | 32 ++++---- py/tests/__init__.py | 0 py/{dnadesign => tests}/data/example.fasta | 0 py/{dnadesign => tests}/test_fasta_parser.py | 2 +- 7 files changed, 100 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 py/Makefile create mode 100644 py/tests/__init__.py rename py/{dnadesign => tests}/data/example.fasta (100%) rename py/{dnadesign => tests}/test_fasta_parser.py (84%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..07ea954e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,83 @@ +name: Build and Package + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + arch: [amd64, arm64] + exclude: + - os: windows-latest + arch: arm64 + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel + + - name: Build Go shared library + working-directory: ./py + run: | + GOOS=${{ runner.os }} GOARCH=${{ matrix.arch }} go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go + env: + CGO_ENABLED: 1 + + - name: Build Python package + working-directory: ./py + run: python setup.py sdist bdist_wheel + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: dist-${{ runner.os }}-${{ matrix.arch }} + path: py/dist/ + + publish: + needs: build + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install twine + + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + path: dist + + - name: Publish to PyPI + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: twine upload dist/**/*/dist/* diff --git a/py/Makefile b/py/Makefile deleted file mode 100644 index 9a90561c..00000000 --- a/py/Makefile +++ /dev/null @@ -1,30 +0,0 @@ -.PHONY: all clean build - -# List of architectures and operating systems to support -ARCHS = amd64 arm64 -OSES = linux darwin windows - -# Mapping Go architectures to Zig architectures -ZIG_ARCHS_amd64 = x86_64 -ZIG_ARCHS_arm64 = aarch64 - -# Mapping Go operating systems to Zig targets -ZIG_OSES_linux = linux-gnu -ZIG_OSES_darwin = macos -ZIG_OSES_windows = windows-gnu - -# Default target -all: build - -# Build libraries for all architecture and OS combinations -build: - $(foreach os,$(OSES),\ - $(foreach arch,$(ARCHS),\ - CC="zig cc -target $(ZIG_ARCHS_$(arch))-$(ZIG_OSES_$(os))" \ - CGO_ENABLED=1 GOOS=$(os) GOARCH=$(arch) go build -o dnadesign/lib/libdnadesign_$(os)_$(arch).so -buildmode=c-shared lib.go;\ - )\ - ) - -clean: - rm -f dnadesign/lib/libdnadesign_*.so - diff --git a/py/README.md b/py/README.md index 00e05e70..fa3214ca 100644 --- a/py/README.md +++ b/py/README.md @@ -4,3 +4,4 @@ To deploy your package: 3. Use `twine` to upload the distribution files to PyPI. Simple compilation: `go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go` +`python3 setup.py bdist_wheel --plat-name manylinux1_x86_64` diff --git a/py/setup.py b/py/setup.py index 1850cc28..1c0b41cf 100644 --- a/py/setup.py +++ b/py/setup.py @@ -1,26 +1,24 @@ -from setuptools import setup, find_packages -from setuptools.command.install import install -import platform import os +import platform +from setuptools import setup, find_packages -class CustomInstall(install): - def run(self): - # Install the appropriate shared library based on the user's platform - system = platform.system().lower() - arch = 'amd64' if platform.machine() in ['x86_64', 'AMD64'] else 'arm64' - lib_file = f'dnadesign/lib/libdnadesign_{system}_{arch}.so' - if not os.path.exists(lib_file): - raise FileNotFoundError(f"Could not find required library: {lib_file}") - os.rename(lib_file, 'dnadesign/libdnadesign.so') - install.run(self) +def get_shared_lib_ext(): + if platform.system() == "Darwin": + return ".dylib" + elif platform.system() == "Windows": + return ".dll" + else: + return ".so" setup( name='dnadesign', version='0.1.0', packages=find_packages(), - cmdclass={'install': CustomInstall}, - package_data={'dnadesign': ['libdnadesign_*.so']}, + package_data={'dnadesign': ['definitions.h', 'libdnadesign.h', "libdnadesign" + get_shared_lib_ext()]}, include_package_data=True, - zip_safe=False + zip_safe=False, + author='Keoni Gandall', + author_email='koeng101@gmail.com', + description='Python bindings for dnadesign', + url='https://github.com/koeng101/dnadesign' ) - diff --git a/py/tests/__init__.py b/py/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/py/dnadesign/data/example.fasta b/py/tests/data/example.fasta similarity index 100% rename from py/dnadesign/data/example.fasta rename to py/tests/data/example.fasta diff --git a/py/dnadesign/test_fasta_parser.py b/py/tests/test_fasta_parser.py similarity index 84% rename from py/dnadesign/test_fasta_parser.py rename to py/tests/test_fasta_parser.py index eae8f92a..12ec3c94 100644 --- a/py/dnadesign/test_fasta_parser.py +++ b/py/tests/test_fasta_parser.py @@ -1,6 +1,6 @@ import pytest import os -from .fasta_parser import parse_fasta_from_c_file, parse_fasta_from_c_string, FastaRecord +from dnadesign.fasta_parser import parse_fasta_from_c_file, parse_fasta_from_c_string, FastaRecord def test_parse_fasta_from_c_file(): current_dir = os.path.dirname(__file__) From a39043b12ffc914d58b3d5073f9100628d25c594 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 17:08:40 +0000 Subject: [PATCH 09/35] build.yml --- .github/workflows/build.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 07ea954e..6c69c39c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,12 +24,12 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.17 + go-version: '1.22' - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: '3.10' - name: Install dependencies run: | @@ -39,7 +39,11 @@ jobs: - name: Build Go shared library working-directory: ./py run: | - GOOS=${{ runner.os }} GOARCH=${{ matrix.arch }} go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go + if [ "${{ runner.os }}" = "Linux" ] && [ "${{ matrix.arch }}" = "arm64" ]; then + GOOS=${{ runner.os }} GOARCH=${{ matrix.arch }} go build -o dnadesign/libdnadesign.so -buildmode=pie lib.go + else + GOOS=${{ runner.os }} GOARCH=${{ matrix.arch }} go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go + fi env: CGO_ENABLED: 1 @@ -64,7 +68,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: '3.10' - name: Install dependencies run: | From 76f1f819056f4dbacd4676db20c5734558ada851 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 17:10:47 +0000 Subject: [PATCH 10/35] cgo enable --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6c69c39c..fe46d9ab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,9 +40,9 @@ jobs: working-directory: ./py run: | if [ "${{ runner.os }}" = "Linux" ] && [ "${{ matrix.arch }}" = "arm64" ]; then - GOOS=${{ runner.os }} GOARCH=${{ matrix.arch }} go build -o dnadesign/libdnadesign.so -buildmode=pie lib.go + CGO_ENABLED=1 GOOS=${{ runner.os }} GOARCH=${{ matrix.arch }} go build -o dnadesign/libdnadesign.so -buildmode=pie lib.go else - GOOS=${{ runner.os }} GOARCH=${{ matrix.arch }} go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go + CGO_ENABLED=1 GOOS=${{ runner.os }} GOARCH=${{ matrix.arch }} go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go fi env: CGO_ENABLED: 1 From 508e6fd4b21644fd89cac12317ebe15e54489c26 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 17:21:39 +0000 Subject: [PATCH 11/35] use zig --- .github/workflows/build.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe46d9ab..a8789cf2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,11 +10,8 @@ jobs: build: strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest] arch: [amd64, arm64] - exclude: - - os: windows-latest - arch: arm64 runs-on: ${{ matrix.os }} @@ -36,13 +33,22 @@ jobs: python -m pip install --upgrade pip pip install setuptools wheel + - name: Install Zig + uses: goto-bus-stop/setup-zig@v2 + with: + version: 0.11.0 + - name: Build Go shared library working-directory: ./py run: | if [ "${{ runner.os }}" = "Linux" ] && [ "${{ matrix.arch }}" = "arm64" ]; then - CGO_ENABLED=1 GOOS=${{ runner.os }} GOARCH=${{ matrix.arch }} go build -o dnadesign/libdnadesign.so -buildmode=pie lib.go + CC="zig cc -target aarch64-linux-gnu" CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go + elif [ "${{ runner.os }}" = "Linux" ] && [ "${{ matrix.arch }}" = "amd64" ]; then + CC="zig cc -target x86_64-linux-gnu" GOOS=linux GOARCH=amd64 go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go + elif [ "${{ runner.os }}" = "macOS" ] && [ "${{ matrix.arch }}" = "arm64" ]; then + CC="zig cc -target aarch64-macos" CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go else - CGO_ENABLED=1 GOOS=${{ runner.os }} GOARCH=${{ matrix.arch }} go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go + CC="zig cc -target x86_64-macos" CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go fi env: CGO_ENABLED: 1 From c9ca991f95f99afab866eac3285353ae599a875d Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 17:28:02 +0000 Subject: [PATCH 12/35] build --- .github/workflows/build.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a8789cf2..1cf7ce1e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,6 +38,12 @@ jobs: with: version: 0.11.0 + - name: Set up Xcode + if: runner.os == 'macOS' + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: latest-stable + - name: Build Go shared library working-directory: ./py run: | @@ -45,10 +51,13 @@ jobs: CC="zig cc -target aarch64-linux-gnu" CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go elif [ "${{ runner.os }}" = "Linux" ] && [ "${{ matrix.arch }}" = "amd64" ]; then CC="zig cc -target x86_64-linux-gnu" GOOS=linux GOARCH=amd64 go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go - elif [ "${{ runner.os }}" = "macOS" ] && [ "${{ matrix.arch }}" = "arm64" ]; then - CC="zig cc -target aarch64-macos" CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go - else - CC="zig cc -target x86_64-macos" CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go + elif [ "${{ runner.os }}" = "macOS" ]; then + SDK_PATH=$(xcrun --show-sdk-path) + if [ "${{ matrix.arch }}" = "arm64" ]; then + CC="zig cc -target aarch64-macos -isysroot $SDK_PATH" CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go + else + CC="zig cc -target x86_64-macos -isysroot $SDK_PATH" CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go + fi fi env: CGO_ENABLED: 1 From afe77d1c1182a06f12817d7ab564922fc41a45a4 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 17:38:11 +0000 Subject: [PATCH 13/35] remove zig from mac osx --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1cf7ce1e..4dfe8411 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -54,9 +54,9 @@ jobs: elif [ "${{ runner.os }}" = "macOS" ]; then SDK_PATH=$(xcrun --show-sdk-path) if [ "${{ matrix.arch }}" = "arm64" ]; then - CC="zig cc -target aarch64-macos -isysroot $SDK_PATH" CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go + CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go else - CC="zig cc -target x86_64-macos -isysroot $SDK_PATH" CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go + CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go fi fi env: From 74955b49508eefe764aa14fdfb24817917e28c04 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 17:45:38 +0000 Subject: [PATCH 14/35] add test step --- .github/workflows/build.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4dfe8411..11c2dc59 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,6 +66,17 @@ jobs: working-directory: ./py run: python setup.py sdist bdist_wheel + - name: Install Python package and dependencies + working-directory: ./py + run: | + python -m pip install --upgrade pip + pip install . + pip install pytest + + - name: Run pytest + working-directory: ./py + run: pytest + - name: Upload artifacts uses: actions/upload-artifact@v2 with: From f02499a3d533ab83d5604d775395211ff382a0a9 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 17:46:59 +0000 Subject: [PATCH 15/35] add cffi --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 11c2dc59..2bc1f43c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel + pip install setuptools wheel cffi - name: Install Zig uses: goto-bus-stop/setup-zig@v2 From 9fd42641314029ad1a9a0b5d2d79c6014b33ef1a Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 17:51:35 +0000 Subject: [PATCH 16/35] Updated cffi bindings for proper darwin support --- py/dnadesign/cffi_bindings.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/py/dnadesign/cffi_bindings.py b/py/dnadesign/cffi_bindings.py index 17f63914..0db6260e 100644 --- a/py/dnadesign/cffi_bindings.py +++ b/py/dnadesign/cffi_bindings.py @@ -15,7 +15,16 @@ # Build the path to definitions.h and libdnadesign relative to the current script definitions_path = os.path.join(current_dir, 'definitions.h') -lib_path = os.path.join(current_dir, 'libdnadesign.so') + +# Determine the correct library name based on the operating system +if sys.platform.startswith('win'): + lib_name = 'libdnadesign.dll' +elif sys.platform.startswith('darwin'): + lib_name = 'libdnadesign.dylib' +else: + lib_name = 'libdnadesign.so' + +lib_path = os.path.join(current_dir, lib_name) # Read the C declarations from an external file with open(definitions_path, 'r') as f: From 2bb83b9bd243beb40413e2c3339a7d9657f8dfd6 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 17:55:23 +0000 Subject: [PATCH 17/35] Updated with amd/arm for mac --- .github/workflows/build.yml | 11 ++++------- py/dnadesign/cffi_bindings.py | 7 +++++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2bc1f43c..3e91bd80 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,13 +51,10 @@ jobs: CC="zig cc -target aarch64-linux-gnu" CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go elif [ "${{ runner.os }}" = "Linux" ] && [ "${{ matrix.arch }}" = "amd64" ]; then CC="zig cc -target x86_64-linux-gnu" GOOS=linux GOARCH=amd64 go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go - elif [ "${{ runner.os }}" = "macOS" ]; then - SDK_PATH=$(xcrun --show-sdk-path) - if [ "${{ matrix.arch }}" = "arm64" ]; then - CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go - else - CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go - fi + elif [ "${{ runner.os }}" = "macOS" ] && [ "${{ matrix.arch }}" = "arm64" ]; then + CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -o dnadesign/libdnadesign_arm64.dylib -buildmode=c-shared lib.go + elif [ "${{ runner.os }}" = "macOS" ] && [ "${{ matrix.arch }}" = "amd64" ]; then + CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -o dnadesign/libdnadesign_amd64.dylib -buildmode=c-shared lib.go fi env: CGO_ENABLED: 1 diff --git a/py/dnadesign/cffi_bindings.py b/py/dnadesign/cffi_bindings.py index 0db6260e..63dbfbdf 100644 --- a/py/dnadesign/cffi_bindings.py +++ b/py/dnadesign/cffi_bindings.py @@ -16,11 +16,14 @@ # Build the path to definitions.h and libdnadesign relative to the current script definitions_path = os.path.join(current_dir, 'definitions.h') -# Determine the correct library name based on the operating system +# Determine the correct library name based on the operating system and architecture if sys.platform.startswith('win'): lib_name = 'libdnadesign.dll' elif sys.platform.startswith('darwin'): - lib_name = 'libdnadesign.dylib' + if platform.machine() == 'arm64': + lib_name = 'libdnadesign_arm64.dylib' + else: + lib_name = 'libdnadesign_amd64.dylib' else: lib_name = 'libdnadesign.so' From b1c5f244b2161cb69986e79d73fd57987eba53f4 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 17:56:54 +0000 Subject: [PATCH 18/35] add platform --- py/dnadesign/cffi_bindings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/py/dnadesign/cffi_bindings.py b/py/dnadesign/cffi_bindings.py index 63dbfbdf..ef0b13d5 100644 --- a/py/dnadesign/cffi_bindings.py +++ b/py/dnadesign/cffi_bindings.py @@ -1,4 +1,5 @@ from cffi import FFI +import platform import os import sys From 75bd0c3d1a5dc7df50916bf9509834787970add1 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:00:01 +0000 Subject: [PATCH 19/35] build --- .github/workflows/build.yml | 6 +++--- py/dnadesign/cffi_bindings.py | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3e91bd80..42e8db24 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,13 +48,13 @@ jobs: working-directory: ./py run: | if [ "${{ runner.os }}" = "Linux" ] && [ "${{ matrix.arch }}" = "arm64" ]; then - CC="zig cc -target aarch64-linux-gnu" CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go + CC="zig cc -target aarch64-linux-gnu" GOOS=linux GOARCH=arm64 go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go elif [ "${{ runner.os }}" = "Linux" ] && [ "${{ matrix.arch }}" = "amd64" ]; then CC="zig cc -target x86_64-linux-gnu" GOOS=linux GOARCH=amd64 go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go elif [ "${{ runner.os }}" = "macOS" ] && [ "${{ matrix.arch }}" = "arm64" ]; then - CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -o dnadesign/libdnadesign_arm64.dylib -buildmode=c-shared lib.go + CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go elif [ "${{ runner.os }}" = "macOS" ] && [ "${{ matrix.arch }}" = "amd64" ]; then - CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -o dnadesign/libdnadesign_amd64.dylib -buildmode=c-shared lib.go + CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -o dnadesign/libdnadesign.dylib -buildmode=c-shared lib.go fi env: CGO_ENABLED: 1 diff --git a/py/dnadesign/cffi_bindings.py b/py/dnadesign/cffi_bindings.py index ef0b13d5..bc52f69e 100644 --- a/py/dnadesign/cffi_bindings.py +++ b/py/dnadesign/cffi_bindings.py @@ -21,10 +21,7 @@ if sys.platform.startswith('win'): lib_name = 'libdnadesign.dll' elif sys.platform.startswith('darwin'): - if platform.machine() == 'arm64': - lib_name = 'libdnadesign_arm64.dylib' - else: - lib_name = 'libdnadesign_amd64.dylib' + lib_name = 'libdnadesign.dylib' else: lib_name = 'libdnadesign.so' From 0cf23aebb330c1bf378b10c316cb3ec1303b9453 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:04:43 +0000 Subject: [PATCH 20/35] build after test --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 42e8db24..fc265d11 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,10 +59,6 @@ jobs: env: CGO_ENABLED: 1 - - name: Build Python package - working-directory: ./py - run: python setup.py sdist bdist_wheel - - name: Install Python package and dependencies working-directory: ./py run: | @@ -74,6 +70,10 @@ jobs: working-directory: ./py run: pytest + - name: Build Python package + working-directory: ./py + run: python setup.py sdist bdist_wheel + - name: Upload artifacts uses: actions/upload-artifact@v2 with: From 13d9c8127440b74e3497dfa1dbed7304fe7013ab Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:08:18 +0000 Subject: [PATCH 21/35] build --- .github/workflows/build.yml | 4 ++++ py/dnadesign/cffi_bindings.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fc265d11..c0595ab2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,6 +59,10 @@ jobs: env: CGO_ENABLED: 1 + - name: List directory contents + working-directory: ./py/dnadesign + run: ls -l + - name: Install Python package and dependencies working-directory: ./py run: | diff --git a/py/dnadesign/cffi_bindings.py b/py/dnadesign/cffi_bindings.py index bc52f69e..506e17fd 100644 --- a/py/dnadesign/cffi_bindings.py +++ b/py/dnadesign/cffi_bindings.py @@ -27,6 +27,11 @@ lib_path = os.path.join(current_dir, lib_name) +print(f"Platform: {sys.platform}") +print(f"Machine: {platform.machine()}") +print(f"Looking for library: {lib_path}") +print(f"Directory contents: {os.listdir(os.path.dirname(__file__))}") + # Read the C declarations from an external file with open(definitions_path, 'r') as f: ffi.cdef(f.read()) From dcd33b64a58701a5e28e82c37c5533a6f90c262c Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:18:45 +0000 Subject: [PATCH 22/35] downgrade mac --- .github/workflows/build.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c0595ab2..7fa69e32 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,9 +9,15 @@ on: jobs: build: strategy: - matrix: - os: [ubuntu-latest, macos-latest] - arch: [amd64, arm64] + include: + - os: ubuntu-latest + arch: amd64 + - os: ubuntu-latest + arch: arm64 + - os: macos-latest # This will be ARM64 + arch: arm64 + - os: macos-13 # This will be AMD64 + arch: amd64 runs-on: ${{ matrix.os }} From 1b57adf898d4b3604e272925aae1327903a8fda4 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:20:14 +0000 Subject: [PATCH 23/35] build indents? --- .github/workflows/build.yml | 5 ++++- py/dnadesign/cffi_bindings.py | 9 +-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7fa69e32..5d413a13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,11 +9,14 @@ on: jobs: build: strategy: - include: + matrix: + include: - os: ubuntu-latest arch: amd64 - os: ubuntu-latest arch: arm64 + - os: windows-latest + arch: amd64 - os: macos-latest # This will be ARM64 arch: arm64 - os: macos-13 # This will be AMD64 diff --git a/py/dnadesign/cffi_bindings.py b/py/dnadesign/cffi_bindings.py index 506e17fd..6415a2c0 100644 --- a/py/dnadesign/cffi_bindings.py +++ b/py/dnadesign/cffi_bindings.py @@ -18,20 +18,13 @@ definitions_path = os.path.join(current_dir, 'definitions.h') # Determine the correct library name based on the operating system and architecture -if sys.platform.startswith('win'): - lib_name = 'libdnadesign.dll' -elif sys.platform.startswith('darwin'): +if sys.platform.startswith('darwin'): lib_name = 'libdnadesign.dylib' else: lib_name = 'libdnadesign.so' lib_path = os.path.join(current_dir, lib_name) -print(f"Platform: {sys.platform}") -print(f"Machine: {platform.machine()}") -print(f"Looking for library: {lib_path}") -print(f"Directory contents: {os.listdir(os.path.dirname(__file__))}") - # Read the C declarations from an external file with open(definitions_path, 'r') as f: ffi.cdef(f.read()) From c79b670e72694853684bf8a58e0c69fafdfad785 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:24:29 +0000 Subject: [PATCH 24/35] finally --- py/dnadesign/cffi_bindings.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/py/dnadesign/cffi_bindings.py b/py/dnadesign/cffi_bindings.py index 6415a2c0..2940bc95 100644 --- a/py/dnadesign/cffi_bindings.py +++ b/py/dnadesign/cffi_bindings.py @@ -25,6 +25,13 @@ lib_path = os.path.join(current_dir, lib_name) +print(f"Python version: {sys.version}") +print(f"Platform: {sys.platform}") +print(f"Machine: {platform.machine()}") +print(f"Looking for library: {lib_path}") +print(f"Directory contents: {os.listdir(os.path.dirname(__file__))}") +print(f"LD_LIBRARY_PATH: {os.environ.get('LD_LIBRARY_PATH', 'Not set')}") + # Read the C declarations from an external file with open(definitions_path, 'r') as f: ffi.cdef(f.read()) From b0fa017431a85e730d30fdaf3bc4ac10e11623a2 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:28:32 +0000 Subject: [PATCH 25/35] build --- .github/workflows/build.yml | 21 ++++++++------------- py/dnadesign/cffi_bindings.py | 7 ------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5d413a13..25160242 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,8 +15,6 @@ jobs: arch: amd64 - os: ubuntu-latest arch: arm64 - - os: windows-latest - arch: amd64 - os: macos-latest # This will be ARM64 arch: arm64 - os: macos-13 # This will be AMD64 @@ -72,21 +70,18 @@ jobs: working-directory: ./py/dnadesign run: ls -l - - name: Install Python package and dependencies - working-directory: ./py - run: | - python -m pip install --upgrade pip - pip install . - pip install pytest - - - name: Run pytest - working-directory: ./py - run: pytest - - name: Build Python package working-directory: ./py run: python setup.py sdist bdist_wheel + - name: Test wheel in fresh environment + run: | + python -m venv test_env + source test_env/bin/activate + pip install ./py/dist/*.whl + python -c "from dnadesign import fasta_parser; print('Library loaded successfully')" + pytest ./py/tests -v + - name: Upload artifacts uses: actions/upload-artifact@v2 with: diff --git a/py/dnadesign/cffi_bindings.py b/py/dnadesign/cffi_bindings.py index 2940bc95..6415a2c0 100644 --- a/py/dnadesign/cffi_bindings.py +++ b/py/dnadesign/cffi_bindings.py @@ -25,13 +25,6 @@ lib_path = os.path.join(current_dir, lib_name) -print(f"Python version: {sys.version}") -print(f"Platform: {sys.platform}") -print(f"Machine: {platform.machine()}") -print(f"Looking for library: {lib_path}") -print(f"Directory contents: {os.listdir(os.path.dirname(__file__))}") -print(f"LD_LIBRARY_PATH: {os.environ.get('LD_LIBRARY_PATH', 'Not set')}") - # Read the C declarations from an external file with open(definitions_path, 'r') as f: ffi.cdef(f.read()) From e30e02c732709673d6309a2a452e0008fe08da76 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:30:16 +0000 Subject: [PATCH 26/35] add dependencies --- py/setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/py/setup.py b/py/setup.py index 1c0b41cf..2427f2ea 100644 --- a/py/setup.py +++ b/py/setup.py @@ -15,6 +15,13 @@ def get_shared_lib_ext(): version='0.1.0', packages=find_packages(), package_data={'dnadesign': ['definitions.h', 'libdnadesign.h', "libdnadesign" + get_shared_lib_ext()]}, + install_requires=[ + "cffi>=1.0.0", + ], + setup_requires=[ + "cffi>=1.0.0", + ], + include_package_data=True, zip_safe=False, author='Keoni Gandall', From 5c9fde1239ccd1b46ec4eddf5553a8cfbdffc742 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:31:48 +0000 Subject: [PATCH 27/35] add pytest --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 25160242..dede783e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,6 +80,7 @@ jobs: source test_env/bin/activate pip install ./py/dist/*.whl python -c "from dnadesign import fasta_parser; print('Library loaded successfully')" + pip install pytest pytest ./py/tests -v - name: Upload artifacts From 0aed0b2de8ab7b85f343561936c6bbbf4bb6f1cc Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:37:44 +0000 Subject: [PATCH 28/35] build without loading first? --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dede783e..f3057025 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -79,7 +79,7 @@ jobs: python -m venv test_env source test_env/bin/activate pip install ./py/dist/*.whl - python -c "from dnadesign import fasta_parser; print('Library loaded successfully')" + #python -c "from dnadesign import fasta_parser; print('Library loaded successfully')" pip install pytest pytest ./py/tests -v From 2c979791a048a0210614c680c09cfa2d1a53e62f Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:40:07 +0000 Subject: [PATCH 29/35] add trace --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f3057025..4501cb3c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -79,9 +79,10 @@ jobs: python -m venv test_env source test_env/bin/activate pip install ./py/dist/*.whl - #python -c "from dnadesign import fasta_parser; print('Library loaded successfully')" + python -c "from dnadesign import fasta_parser; print('Library loaded successfully')" pip install pytest - pytest ./py/tests -v + #pytest ./py/tests -v + gdb -ex "run" -ex "bt full" -ex "quit" --args python -m pytest ./py/tests - - name: Upload artifacts uses: actions/upload-artifact@v2 From d5b2e90dfeb56da8739af06b90db73ce50b5aa24 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:42:10 +0000 Subject: [PATCH 30/35] segmentation --- .github/workflows/build.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4501cb3c..60f8d1fc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,8 +81,20 @@ jobs: pip install ./py/dist/*.whl python -c "from dnadesign import fasta_parser; print('Library loaded successfully')" pip install pytest - #pytest ./py/tests -v - gdb -ex "run" -ex "bt full" -ex "quit" --args python -m pytest ./py/tests - + pytest ./py/tests -v --capture=no + continue-on-error: true + + - name: Debug segmentation fault (macOS) + if: failure() && runner.os == 'macOS' + run: | + lldb -o "run" -o "bt all" -o "quit" -- python -m pytest ./py/tests -v + + - name: Debug segmentation fault (Linux) + if: failure() && runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y gdb + gdb -ex "run" -ex "bt full" -ex "quit" --args python -m pytest ./py/tests -v - name: Upload artifacts uses: actions/upload-artifact@v2 From 799db6e762d80e40bf4d8e891b8a9c079432eac7 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:46:06 +0000 Subject: [PATCH 31/35] add readme --- py/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/py/README.md b/py/README.md index fa3214ca..63b4ad96 100644 --- a/py/README.md +++ b/py/README.md @@ -1,7 +1,7 @@ -To deploy your package: -1. Run `make` to build the shared libraries for all supported platforms. -2. Run `python3 setup.py sdist bdist_wheel` to create the distribution files. -3. Use `twine` to upload the distribution files to PyPI. +# DnaDesign (Python) +This directory contains code for allowing python users to use dnadesign through a shared C library. -Simple compilation: `go build -o dnadesign/libdnadesign.so -buildmode=c-shared lib.go` -`python3 setup.py bdist_wheel --plat-name manylinux1_x86_64` +This is a work-in-progress. Right now, we have only ported the fasta parser. + +### Other platforms +If you have interest in other platforms, like openbsd or freebsd, please add an issue! I'd be happy to add automatic packaging for these alternative platforms if I know someone will use them. From 654ae0d81f713125aaeea3ed81d9cedaae6d1cd7 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:47:16 +0000 Subject: [PATCH 32/35] add docs to main readme --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 7267791b..3273d89c 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,7 @@ DnaDesign is a Go project creating tools for automated genetic design, spanning On the highest level: * `lib` contains core functionality as a go library. * `external` contains functions to work with external bioinformatics command-line interfaces. -* `api` contains an OpenAPI exposing all the major functions of lib. -* `deployment` contains full integration tests and yaml for deploying the DnaDesign API to a k3s cluster. +* `py` contains code to use the dnadesign library in python using a C shared library. ### Detailed repo organization From e19bfaab927484514efa26dcbb2ef161867aa833 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:48:25 +0000 Subject: [PATCH 33/35] bigger note --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 3273d89c..c520d8a9 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,9 @@ On the highest level: * [external/minimap2](https://pkg.go.dev/github.com/koeng101/dnadesign/external/minimap2) contains a function for working with [minimap2](https://github.com/lh3/minimap2) with Go. * [external/samtools](https://pkg.go.dev/github.com/koeng101/dnadesign/external/samtools) contains a function for generating pileup files using [samtools](https://github.com/samtools/samtools) with Go. +## Python + +We have python package, `dnadesign`, which allows python users to use dnadesign. This is a work-in-progress: more documentation coming soon! ## Contributing From 904d6334e554723c8ee2fd7a8ff4649ff9441ea5 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 18:49:17 +0000 Subject: [PATCH 34/35] python in changelog --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c520d8a9..4384cc27 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- Added minimal python packaging [#81](https://github.com/Koeng101/dnadesign/pull/81) - Greatly simplified the Ligate function [#77](https://github.com/Koeng101/dnadesign/pull/77) - Updated barcoding functions to handle edge case of hanging-edge barcodes [#74](https://github.com/Koeng101/dnadesign/pull/74) - Updated megamash to use int instead of uint for minimal Kmer counts (so you can use -1) [#73](https://github.com/Koeng101/dnadesign/pull/73) From 19dfc7755c1b2f5ee788f8d77528d09d4d7e5114 Mon Sep 17 00:00:00 2001 From: Keoni Gandall Date: Sun, 11 Aug 2024 19:02:22 +0000 Subject: [PATCH 35/35] increment version --- py/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/setup.py b/py/setup.py index 2427f2ea..9c837dd0 100644 --- a/py/setup.py +++ b/py/setup.py @@ -12,7 +12,7 @@ def get_shared_lib_ext(): setup( name='dnadesign', - version='0.1.0', + version='0.1.1', packages=find_packages(), package_data={'dnadesign': ['definitions.h', 'libdnadesign.h', "libdnadesign" + get_shared_lib_ext()]}, install_requires=[