Skip to content

Commit

Permalink
feat: update sqlite to use a hand build schema
Browse files Browse the repository at this point in the history 
This is the next iteration on from the basic table layout, there has
been more effort in normalisation.  There are still some outstanding
issues, one improvement would be to consolidate some of the tables.
  • Loading branch information
@ozym
ozym committed Jan 17, 2025
1 parent c19eacc commit cb0cff1
Show file tree
Hide file tree
Showing 34 changed files with 6,739 additions and 1,026 deletions.
36 changes: 21 additions & 15 deletions cmd/deltadb/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,14 @@ and then to examine the file
sqlite3 delta.db
SQLite version 3.46.1 2024-08-13 09:16:08
Enter ".help" for usage hints.
sqlite> .schema Network
CREATE TABLE Network (
Code TEXT PRIMARY KEY,
External TEXT,
Description TEXT,
Restricted TEXT
sqlite> .schema network
CREATE TABLE network (
network_id INTEGER PRIMARY KEY NOT NULL,
network TEXT NOT NULL,
external TEXT NOT NULL,
description TEXT DEFAULT "" NOT NULL,
restricted BOOLEAN DEFAULT false NOT NULL,
UNIQUE (network)
);
sqlite>
```
Expand Down Expand Up @@ -90,16 +92,20 @@ curl -s localhost:8080/network|jq .|head
or

```
curl -s localhost:8080/station/WEL/site/20/sensor/|jq . | head
curl -s localhost:8080/station/WEL|jq . | head
[
{
"make": "Kinemetrics",
"model": "FBA-23",
"serial": "25073",
"start": "1990-10-03T03:04:00Z",
"end": "2003-04-08T00:00:00Z",
"dip": 0,
"azimuth": 0,
"factor": 0,
"station": "WEL",
"network": "NZ",
"name": "Wellington",
"latitude": -41.284047578,
"longitude": 174.768184021,
"elevation": 138,
"datum": "WGS84",
"start": "1916-01-01T00:00:00Z",
...
```

## schema

[![Schema](delta.svg)](delta.svg)
30 changes: 0 additions & 30 deletions cmd/deltadb/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,53 +19,23 @@ func newHandler(db *sql.DB) *http.ServeMux {
mux.HandleFunc("/", homePage)
mux.HandleFunc("GET /network", h.GetNetworks)
mux.HandleFunc("GET /network/", h.GetNetworks)
mux.HandleFunc("GET /network/{code}", h.GetNetworkCode)
mux.HandleFunc("GET /network/{code}/", h.GetNetworkCode)
mux.HandleFunc("GET /network/{code}/station", h.GetNetworkStations)
mux.HandleFunc("GET /network/{code}/station/", h.GetNetworkStations)
mux.HandleFunc("GET /network/{code}/mark", h.GetNetworkMarks)
mux.HandleFunc("GET /network/{code}/mark/", h.GetNetworkMarks)
mux.HandleFunc("GET /network/{code}/mount", h.GetNetworkMounts)
mux.HandleFunc("GET /network/{code}/mount/", h.GetNetworkMounts)
mux.HandleFunc("GET /network/{code}/sample", h.GetNetworkSamples)
mux.HandleFunc("GET /network/{code}/sample/", h.GetNetworkSamples)

mux.HandleFunc("GET /station", h.GetStations)
mux.HandleFunc("GET /station/", h.GetStations)
mux.HandleFunc("GET /station/{code}", h.GetStationCode)
mux.HandleFunc("GET /station/{code}/", h.GetStationCode)
mux.HandleFunc("GET /station/{code}/site", h.GetStationSites)
mux.HandleFunc("GET /station/{code}/site/", h.GetStationSites)
mux.HandleFunc("GET /station/{code}/site/{location}", h.GetStationSiteLocation)
mux.HandleFunc("GET /station/{code}/site/{location}/", h.GetStationSiteLocation)
mux.HandleFunc("GET /station/{code}/site/{location}/sensor", h.GetStationSiteSensors)
mux.HandleFunc("GET /station/{code}/site/{location}/sensor/", h.GetStationSiteSensors)

mux.HandleFunc("GET /site", h.GetSites)
mux.HandleFunc("GET /site/", h.GetSites)
mux.HandleFunc("GET /site/{location}", h.GetSiteLocations)
mux.HandleFunc("GET /site/{location}/", h.GetSiteLocations)

mux.HandleFunc("GET /mark", h.GetMarks)
mux.HandleFunc("GET /mark/", h.GetMarks)
mux.HandleFunc("GET /mark/{code}", h.GetMarkCode)
mux.HandleFunc("GET /mark/{code}/", h.GetMarkCode)

mux.HandleFunc("GET /monument", h.GetMonuments)
mux.HandleFunc("GET /monument/", h.GetMonuments)
mux.HandleFunc("GET /monument/{mark}", h.GetMonumentMark)
mux.HandleFunc("GET /monument/{mark}/", h.GetMonumentMark)

mux.HandleFunc("GET /sample", h.GetSamples)
mux.HandleFunc("GET /sample/", h.GetSamples)
mux.HandleFunc("GET /sample/{code}", h.GetSampleCode)
mux.HandleFunc("GET /sample/{code}/", h.GetSampleCode)
mux.HandleFunc("GET /sample/{code}/point", h.GetSamplePoints)
mux.HandleFunc("GET /sample/{code}/point/", h.GetSamplePoints)
mux.HandleFunc("GET /sample/{code}/point/{location}", h.GetSamplePointLocation)
mux.HandleFunc("GET /sample/{code}/point/{location}/", h.GetSamplePointLocation)
mux.HandleFunc("GET /sample/{code}/point/{location}/sensor", h.GetSamplePointLocationSensors)
mux.HandleFunc("GET /sample/{code}/point/{location}/sensor/", h.GetSamplePointLocationSensors)

mux.HandleFunc("GET /sensor", h.GetSensors)
mux.HandleFunc("GET /sensor/", h.GetSensors)
Expand Down
2 changes: 1 addition & 1 deletion cmd/deltadb/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ func main() {
if settings.db == "" || settings.init {

// insert extra response files
extra := set.KeyValue(settings.response, "Name", "Response", values)
extra := set.KeyValue(settings.response, "Response", "XML", values)

log.Println("initialise database")
start := time.Now()
Expand Down
47 changes: 34 additions & 13 deletions cmd/deltadb/mark.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,53 @@ import (
"encoding/json"
"log"
"net/http"

"github.com/GeoNet/delta/meta/sqlite"
"time"
)

type Mark struct {
Mark string `json:"station"`
Network string `json:"network"`
Igs string `json:"igs"`
Name string `json:"name"`
Latitude float64 `json:"latitude"`
Longitude float64 `json:"longitude"`
Elevation *float64 `json:"elevation,omitempty"`
Datum string `json:"datum"`
Start time.Time `json:"start"`
End time.Time `json:"end"`
}

func (h handler) GetMarks(w http.ResponseWriter, r *http.Request) {

marks, err := sqlite.Marks(r.Context(), h.db)
query := "SELECT mark.mark,network.network,mark.igs,mark.name,mark.latitude,mark.longitude,mark.elevation,datum.datum,mark.start_date,mark.end_date FROM mark, datum, network, mark_network WHERE mark.datum_id = datum.datum_id AND mark.mark_id = mark_network.mark_id AND network.network_id = mark_network.network_id"
stmt, err := h.db.PrepareContext(r.Context(), query)
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
defer stmt.Close()

w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(marks); err != nil {
log.Println(err)
results, err := stmt.QueryContext(r.Context())
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
}
defer results.Close()

func (h handler) GetMarkCode(w http.ResponseWriter, r *http.Request) {

code := r.PathValue("code")
marks := make([]Mark, 0)
for results.Next() {
var mark Mark
if err := results.Scan(&mark.Mark, &mark.Network, &mark.Igs, &mark.Name, &mark.Latitude, &mark.Longitude, &mark.Elevation, &mark.Datum, &mark.Start, &mark.End); err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
marks = append(marks, mark)
}

marks, err := sqlite.Marks(r.Context(), h.db, sqlite.Code(code))
if err != nil {
if err = results.Err(); err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
Expand Down
48 changes: 35 additions & 13 deletions cmd/deltadb/monument.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,54 @@ import (
"encoding/json"
"log"
"net/http"

"github.com/GeoNet/delta/meta/sqlite"
"time"
)

type Monument struct {
Mark string `json:"mark"`
DomesNumber string `json:"domes_number,omitempty"`
MarkType string `json:"mark_type"`
Type string `json:"type"`
GroundRelationship float64 `json:"ground_relationship"`
FoundationType string `json:"foundation_type"`
FoundationDepth float64 `json:"foundation_depth"`
Bedrock string `json:"bedrock,omitempty"`
Geology string `json:"geology,omitempty"`
Start time.Time `json:"start"`
End time.Time `json:"end"`
}

func (h handler) GetMonuments(w http.ResponseWriter, r *http.Request) {

monuments, err := sqlite.Monuments(r.Context(), h.db)
query := "SELECT mark.mark,monument.domes_number,mark_type.mark_type,monument_type.monument_type,monument.ground_relationship,foundation_type.foundation_type,monument.foundation_depth,bedrock.bedrock,geology.geology,monument.start_date,monument.end_date FROM mark, monument, mark_type, monument_type, foundation_type, bedrock, geology WHERE mark.mark_id = monument.mark_id AND monument.mark_type_id = mark_type.mark_type_id AND monument.monument_type_id = monument_type.monument_type_id AND monument.foundation_type_id = foundation_type.foundation_type_id AND monument.bedrock_id = bedrock.bedrock_id AND monument.geology_id = geology.geology_id"
stmt, err := h.db.PrepareContext(r.Context(), query)
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
defer stmt.Close()

w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(monuments); err != nil {
log.Println(err)
results, err := stmt.QueryContext(r.Context())
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
}
defer results.Close()

func (h handler) GetMonumentMark(w http.ResponseWriter, r *http.Request) {

mark := r.PathValue("mark")
monuments := make([]Monument, 0)
for results.Next() {
var monument Monument
if err := results.Scan(&monument.Mark, &monument.DomesNumber, &monument.MarkType, &monument.Type, &monument.GroundRelationship, &monument.FoundationType, &monument.FoundationDepth, &monument.Bedrock, &monument.Geology, &monument.Start, &monument.End); err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
monuments = append(monuments, monument)
}

monuments, err := sqlite.Monuments(r.Context(), h.db, sqlite.Mark(mark))
if err != nil {
if err = results.Err(); err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
Expand Down
100 changes: 21 additions & 79 deletions cmd/deltadb/network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,112 +4,54 @@ import (
"encoding/json"
"log"
"net/http"

"github.com/GeoNet/delta/meta/sqlite"
)

func (h handler) GetNetworks(w http.ResponseWriter, r *http.Request) {

networks, err := sqlite.Networks(r.Context(), h.db)
query := "SELECT network,external,description,restricted FROM network"
stmt, err := h.db.PrepareContext(r.Context(), query)
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
defer stmt.Close()

w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(networks); err != nil {
log.Println(err)
}
}

func (h handler) GetNetworkCode(w http.ResponseWriter, r *http.Request) {

code := r.PathValue("code")

networks, err := sqlite.Networks(r.Context(), h.db, sqlite.Code(code))
results, err := stmt.QueryContext(r.Context())
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
defer results.Close()

w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(networks); err != nil {
log.Println(err)
}
}

func (h handler) GetNetworkStations(w http.ResponseWriter, r *http.Request) {

code := r.PathValue("code")

stations, err := sqlite.Stations(r.Context(), h.db, sqlite.Network(code))
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
type Network struct {
Network string `json:"network"`
External string `json:"external"`
Description string `json:"description"`
Restricted bool `json:"restricted,omitempty"`
}

w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(stations); err != nil {
log.Println(err)
networks := make([]Network, 0)
for results.Next() {
var network Network
if err := results.Scan(&network.Network, &network.External, &network.Description, &network.Restricted); err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
networks = append(networks, network)
}
}

func (h handler) GetNetworkMarks(w http.ResponseWriter, r *http.Request) {

code := r.PathValue("code")

marks, err := sqlite.Marks(r.Context(), h.db, sqlite.Network(code))
if err != nil {
if err = results.Err(); err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}

w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(marks); err != nil {
log.Println(err)
}
}

func (h handler) GetNetworkSamples(w http.ResponseWriter, r *http.Request) {

code := r.PathValue("code")

samples, err := sqlite.Samples(r.Context(), h.db, sqlite.Network(code))
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}

w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(samples); err != nil {
log.Println(err)
}
}

func (h handler) GetNetworkMounts(w http.ResponseWriter, r *http.Request) {

code := r.PathValue("code")

mounts, err := sqlite.Mounts(r.Context(), h.db, sqlite.Network(code))
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}

w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(mounts); err != nil {
if err := json.NewEncoder(w).Encode(networks); err != nil {
log.Println(err)
}
}
Loading

0 comments on commit cb0cff1

Please sign in to comment.