From 1aca135282c7b59f2ec12ed6089aa55550ed411b Mon Sep 17 00:00:00 2001 From: Greg Martin Date: Tue, 24 Oct 2023 12:43:15 +0000 Subject: [PATCH] Merging my changes from 0.9.x to 0.10.x --- .github/workflows/release.yaml | 5 +- CHANGELOG.md | 72 ++++ src/index.rs | 130 ++++++ src/index/updater.rs | 7 + src/index/updater/inscription_updater.rs | 6 + src/options.rs | 28 +- src/subcommand.rs | 8 + src/subcommand/children.rs | 16 + src/subcommand/preview.rs | 2 + src/subcommand/server.rs | 495 ++++++++++++++++++++++- src/subcommand/transfer.rs | 45 +++ src/subcommand/wallet/inscribe.rs | 22 +- src/subcommand/wallet/send.rs | 28 +- 13 files changed, 844 insertions(+), 20 deletions(-) create mode 100644 src/subcommand/children.rs create mode 100644 src/subcommand/transfer.rs diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 374d8f0b1b..59936fa6f0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,6 +9,9 @@ defaults: run: shell: bash +permissions: + contents: write + jobs: release: strategy: @@ -54,7 +57,7 @@ jobs: - name: Release Type id: release-type run: | - if [[ ${{ github.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+$ ]]; then + if [[ ${{ github.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+(-gms?[0-9]+)?$ ]]; then echo ::set-output name=value::release else echo ::set-output name=value::prerelease diff --git a/CHANGELOG.md b/CHANGELOG.md index 764c82a74f..90644ec060 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,78 @@ Changelog - Format rune supply using divisibility (#2509) - Add pre-alpha unstable incomplete half-baked rune index (#2491) +[0.9.0-gm5](https://github.com/ordinals/ord/releases/tag/0.9.0-gm5) - 2023-10-21 +-------------------------------------------------------------------------------- + +### Added + +- Add `/outputs` endpoint to fetch details for multiple outputs per request. + +[0.9.0-gm4](https://github.com/ordinals/ord/releases/tag/0.9.0-gm4) - 2023-10-18 +-------------------------------------------------------------------------------- + +### Added + +- Add `/transfers//` and `/transfers///` endpoints to allow pagination. + +[0.9.0-gm3](https://github.com/ordinals/ord/releases/tag/0.9.0-gm3) - 2023-10-10 +-------------------------------------------------------------------------------- + +### Changed + +- Modify the /ranges endpoint to group the ranges by output. + +[0.9.0-gm2](https://github.com/ordinals/ord/releases/tag/0.9.0-gm2) - 2023-10-10 +-------------------------------------------------------------------------------- + +### Changed + +- Fix github releases. + +[0.9.0-gm1](https://github.com/ordinals/ord/releases/tag/0.9.0-gm1) - 2023-10-10 +-------------------------------------------------------------------------------- + +### Added + +- Add `--ignore-descriptors` flag to allow ord to work with non-ord wallets. + +[0.9.0-gms4](https://github.com/ordinals/ord/releases/tag/0.9.0-gms4) - 2023-09-18 +---------------------------------------------------------------------------------- + +### Added + +- Speed up `/transfers/` endpoint and don't block while running it. +- Add `application/cbor` media type with extension `.cbor` (#2446) +- Add --utxo flag to allow the use of unconfirmed outputs. +- Add --coin-control flag to limit which outputs can be spent. +- Add `/ranges` endpoint for looking up the sat ranges for a batch of outputs. + +[0.9.0-gms3](https://github.com/ordinals/ord/releases/tag/0.9.0-gms3) - 2023-09-12 +---------------------------------------------------------------------------------- + +### Added + +- Add subcommand `children` to list all the child/parent pairs + +[0.9.0-gms2](https://github.com/ordinals/ord/releases/tag/0.9.0-gms2) - 2023-09-11 +---------------------------------------------------------------------------------- + +### Added + +- Add `parent` and `children` to `/inscriptions_json/` endpoint + +[0.9.0-gms1](https://github.com/ordinals/ord/releases/tag/0.9.0-gms1) - 2023-09-11 +---------------------------------------------------------------------------------- + +### Added + +- Add `/inscriptions_json/` endpoint +- Add `/transfers/` endpoint +- Add `/stats/` endpoint +- Only index blocks when new blocks exist and the height limit isn't reached +- Add `--no-progress-bar` flag to inhibit the display of the progress bar +- Add server request logging + [0.9.0](https://github.com/ordinals/ord/releases/tag/0.9.0) - 2023-09-11 ------------------------------------------------------------------------ diff --git a/src/index.rs b/src/index.rs index 2ea43b8898..86a3a1388a 100644 --- a/src/index.rs +++ b/src/index.rs @@ -53,6 +53,7 @@ macro_rules! define_multimap_table { define_multimap_table! { INSCRIPTION_ID_TO_CHILDREN, &InscriptionIdValue, &InscriptionIdValue } define_multimap_table! { SATPOINT_TO_INSCRIPTION_ID, &SatPointValue, &InscriptionIdValue } define_multimap_table! { SAT_TO_INSCRIPTION_ID, u64, &InscriptionIdValue } +define_multimap_table! { HEIGHT_TO_INSCRIPTION_ID, u64, &InscriptionIdValue } define_table! { HEIGHT_TO_BLOCK_HASH, u64, &BlockHashValue } define_table! { HEIGHT_TO_LAST_SEQUENCE_NUMBER, u64, u64 } define_table! { INSCRIPTION_ID_TO_INSCRIPTION_ENTRY, &InscriptionIdValue, InscriptionEntryValue } @@ -157,6 +158,7 @@ pub(crate) struct Index { height_limit: Option, index_runes: bool, index_sats: bool, + no_progress_bar: bool, options: Options, path: PathBuf, unrecoverably_reorged: AtomicBool, @@ -269,6 +271,7 @@ impl Index { tx.open_multimap_table(INSCRIPTION_ID_TO_CHILDREN)?; tx.open_multimap_table(SATPOINT_TO_INSCRIPTION_ID)?; tx.open_multimap_table(SAT_TO_INSCRIPTION_ID)?; + tx.open_multimap_table(HEIGHT_TO_INSCRIPTION_ID)?; tx.open_table(HEIGHT_TO_BLOCK_HASH)?; tx.open_table(HEIGHT_TO_LAST_SEQUENCE_NUMBER)?; tx.open_table(INSCRIPTION_ID_TO_INSCRIPTION_ENTRY)?; @@ -318,6 +321,7 @@ impl Index { first_inscription_height: options.first_inscription_height(), genesis_block_coinbase_transaction, height_limit: options.height_limit, + no_progress_bar: options.no_progress_bar, options: options.clone(), index_runes, index_sats, @@ -794,6 +798,23 @@ impl Index { .collect() } + pub(crate) fn get_inscription_ids_by_height(&self, height: u64) -> Result> { + let mut ret = Vec::new(); + for range in self + .database + .begin_read()? + .open_multimap_table(HEIGHT_TO_INSCRIPTION_ID)? + .range::<&u64>(&height..&(height + 1))? + { + let (_, ids) = range?; + for id in ids { + ret.push(Entry::load(*id?.value())); + } + } + + Ok(ret) + } + pub(crate) fn get_inscription_ids_by_sat(&self, sat: Sat) -> Result> { let rtx = &self.database.begin_read()?; @@ -1080,6 +1101,18 @@ impl Index { } } + pub(crate) fn ranges(&self, outpoint: OutPoint) -> Result> { + match self.list_inner(outpoint.store())? { + Some(sat_ranges) => + Ok(sat_ranges + .chunks_exact(11) + .map(|chunk| SatRange::load(chunk.try_into().unwrap())) + .collect(), + ), + None => Err(anyhow!("no ranges")), + } + } + pub(crate) fn block_time(&self, height: Height) -> Result { let height = height.n(); @@ -1265,6 +1298,103 @@ impl Index { ) } + pub(crate) fn delete_transfer_log(&self) -> Result { + let wtx = self.database.begin_write().unwrap(); + wtx.delete_multimap_table(HEIGHT_TO_INSCRIPTION_ID)?; + Ok(wtx.commit()?) + } + + pub(crate) fn trim_transfer_log(&self, height: u64) -> Result { + let wtx = self.begin_write()?; + for pair in self + .database + .begin_read()? + .open_multimap_table(HEIGHT_TO_INSCRIPTION_ID)? + .range(..height)? + { + wtx + .open_multimap_table(HEIGHT_TO_INSCRIPTION_ID)? + .remove_all(pair?.0.value())?; + } + Ok(wtx.commit()?) + } + + pub(crate) fn show_transfer_log_stats(&self) -> Result<(u64, Option, Option)> { + let rtx = self.database.begin_read().unwrap(); + let table = rtx.open_multimap_table(HEIGHT_TO_INSCRIPTION_ID)?; + let mut iter = table.iter()?; + + let rows = table.len()?; + + let first = iter + .next() + .and_then(|result| result.ok()) + .map(|(height, _id)| height.value()); + + let last = iter + .next_back() + .and_then(|result| result.ok()) + .map(|(height, _id)| height.value()); + + if first.is_none() { + Ok((rows, None, None)) + } else if last.is_none() { + Ok((rows, first, first)) + } else { + Ok((rows, first, last)) + } + } + + pub(crate) fn get_children(&self) -> Result<()> { + for range in self + .database + .begin_read()? + .open_multimap_table(INSCRIPTION_ID_TO_CHILDREN)? + .iter()? + { + let (parent, children) = range?; + for child in children { + println!("{} {}", ::load(*parent.value()), ::load(*child?.value())); + } + } + + Ok(()) + } + + pub(crate) fn get_stats(&self) -> Result<(Option, Option, Option)> { + let rtx = self.database.begin_read().unwrap(); + + let height = rtx + .open_table(HEIGHT_TO_BLOCK_HASH)? + .iter()? + .next_back() + .and_then(|result| result.ok()) + .map(|(height, _hash)| height.value()); + + let table = rtx.open_table(INSCRIPTION_NUMBER_TO_INSCRIPTION_ID)?; + let mut iter = table.iter()?; + + let lowest_number = iter + .next() + .and_then(|result| result.ok()) + .map(|(number, _id)| number.value()); + + let highest_number = iter + .next_back() + .and_then(|result| result.ok()) + .map(|(number, _id)| number.value()); + + Ok(( + height, + lowest_number, + if highest_number.is_none() { + lowest_number + } else { + highest_number + }, + )) + } + #[cfg(test)] fn assert_inscription_location( &self, diff --git a/src/index/updater.rs b/src/index/updater.rs index 03f5d51411..8a45a3b599 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -68,6 +68,7 @@ impl<'index> Updater<'_> { )?; let mut progress_bar = if cfg!(test) + || self.index.no_progress_bar || log_enabled!(log::Level::Info) || starting_height <= self.height || integration_test() @@ -82,6 +83,9 @@ impl<'index> Updater<'_> { Some(progress_bar) }; + if starting_height > self.height + && (self.index.height_limit.is_none() || self.index.height_limit.unwrap() > self.height) + { let rx = Self::fetch_blocks_from(self.index, self.height, self.index.index_sats)?; let (mut outpoint_sender, mut value_receiver) = Self::spawn_fetcher(self.index)?; @@ -152,6 +156,7 @@ impl<'index> Updater<'_> { if let Some(progress_bar) = &mut progress_bar { progress_bar.finish_and_clear(); } + } Ok(()) } @@ -392,6 +397,7 @@ impl<'index> Updater<'_> { } let mut height_to_block_hash = wtx.open_table(HEIGHT_TO_BLOCK_HASH)?; + let mut height_to_inscription_id = wtx.open_multimap_table(HEIGHT_TO_INSCRIPTION_ID)?; let mut height_to_last_sequence_number = wtx.open_table(HEIGHT_TO_LAST_SEQUENCE_NUMBER)?; let mut inscription_id_to_inscription_entry = wtx.open_table(INSCRIPTION_ID_TO_INSCRIPTION_ENTRY)?; @@ -427,6 +433,7 @@ impl<'index> Updater<'_> { let mut inscription_updater = InscriptionUpdater::new( self.height, + &mut height_to_inscription_id, &mut inscription_id_to_children, &mut inscription_id_to_satpoint, value_receiver, diff --git a/src/index/updater/inscription_updater.rs b/src/index/updater/inscription_updater.rs index 45deef074c..f9a0c62c9c 100644 --- a/src/index/updater/inscription_updater.rs +++ b/src/index/updater/inscription_updater.rs @@ -24,6 +24,7 @@ enum Origin { pub(super) struct InscriptionUpdater<'a, 'db, 'tx> { flotsam: Vec, height: u64, + height_to_inscription_id: &'a mut MultimapTable<'db, 'tx, u64, &'static InscriptionIdValue>, id_to_children: &'a mut MultimapTable<'db, 'tx, &'static InscriptionIdValue, &'static InscriptionIdValue>, id_to_satpoint: &'a mut Table<'db, 'tx, &'static InscriptionIdValue, &'static SatPointValue>, @@ -48,6 +49,7 @@ pub(super) struct InscriptionUpdater<'a, 'db, 'tx> { impl<'a, 'db, 'tx> InscriptionUpdater<'a, 'db, 'tx> { pub(super) fn new( height: u64, + height_to_inscription_id: &'a mut MultimapTable<'db, 'tx, u64, &'static InscriptionIdValue>, id_to_children: &'a mut MultimapTable< 'db, 'tx, @@ -84,6 +86,7 @@ impl<'a, 'db, 'tx> InscriptionUpdater<'a, 'db, 'tx> { Ok(Self { flotsam: Vec::new(), height, + height_to_inscription_id, id_to_children, id_to_satpoint, value_receiver, @@ -435,6 +438,9 @@ impl<'a, 'db, 'tx> InscriptionUpdater<'a, 'db, 'tx> { let inscription_id = flotsam.inscription_id.store(); let unbound = match flotsam.origin { Origin::Old { old_satpoint } => { + self + .height_to_inscription_id + .insert(&self.height, &inscription_id)?; self.satpoint_to_id.remove_all(&old_satpoint.store())?; false diff --git a/src/options.rs b/src/options.rs index 01ddd31723..f798549497 100644 --- a/src/options.rs +++ b/src/options.rs @@ -49,6 +49,8 @@ pub(crate) struct Options { pub(crate) index_runes_pre_alpha_i_agree_to_get_rekt: bool, #[arg(long, help = "Track location of all satoshis.")] pub(crate) index_sats: bool, + #[arg(long, help = "Inhibit the display of the progress bar while updating the index.")] + pub(crate) no_progress_bar: bool, #[arg(long, short, help = "Use regtest. Equivalent to `--chain regtest`.")] pub(crate) regtest: bool, #[arg(long, help = "Connect to Bitcoin Core RPC at .")] @@ -59,6 +61,8 @@ pub(crate) struct Options { pub(crate) testnet: bool, #[arg(long, default_value = "ord", help = "Use wallet named .")] pub(crate) wallet: String, + #[arg(long, short, help = "Don't check for standard wallet descriptors.")] + pub(crate) ignore_descriptors: bool, #[arg(long, short = 'j', help = "Enable JSON API.")] pub(crate) enable_json_api: bool, } @@ -260,20 +264,22 @@ impl Options { client.load_wallet(&self.wallet)?; } - let descriptors = client.list_descriptors(None)?.descriptors; + if !self.ignore_descriptors { + let descriptors = client.list_descriptors(None)?.descriptors; - let tr = descriptors - .iter() - .filter(|descriptor| descriptor.desc.starts_with("tr(")) - .count(); + let tr = descriptors + .iter() + .filter(|descriptor| descriptor.desc.starts_with("tr(")) + .count(); - let rawtr = descriptors - .iter() - .filter(|descriptor| descriptor.desc.starts_with("rawtr(")) - .count(); + let rawtr = descriptors + .iter() + .filter(|descriptor| descriptor.desc.starts_with("rawtr(")) + .count(); - if tr != 2 || descriptors.len() != 2 + rawtr { - bail!("wallet \"{}\" contains unexpected output descriptors, and does not appear to be an `ord` wallet, create a new wallet with `ord wallet create`", self.wallet); + if tr != 2 || descriptors.len() != 2 + rawtr { + bail!("wallet \"{}\" contains unexpected output descriptors, and does not appear to be an `ord` wallet, create a new wallet with `ord wallet create`", self.wallet); + } } } diff --git a/src/subcommand.rs b/src/subcommand.rs index 93e393dcea..5cc88dbacb 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -1,5 +1,6 @@ use super::*; +pub mod children; pub mod decode; pub mod epochs; pub mod find; @@ -13,10 +14,13 @@ pub mod subsidy; pub mod supply; pub mod teleburn; pub mod traits; +pub mod transfer; pub mod wallet; #[derive(Debug, Parser)] pub(crate) enum Subcommand { + #[command(about = "List all the child inscriptions")] + Children(children::Children), #[command(about = "Decode a transaction")] Decode(decode::Decode), #[command(about = "List the first satoshis of each reward epoch")] @@ -43,6 +47,8 @@ pub(crate) enum Subcommand { Teleburn(teleburn::Teleburn), #[command(about = "Display satoshi traits")] Traits(traits::Traits), + #[command(about = "Modify transfer log table")] + Transfer(transfer::Transfer), #[command(subcommand, about = "Wallet commands")] Wallet(wallet::Wallet), } @@ -50,6 +56,7 @@ pub(crate) enum Subcommand { impl Subcommand { pub(crate) fn run(self, options: Options) -> SubcommandResult { match self { + Self::Children(children) => children.run(options), Self::Decode(decode) => decode.run(), Self::Epochs => epochs::run(), Self::Find(find) => find.run(options), @@ -68,6 +75,7 @@ impl Subcommand { Self::Supply => supply::run(), Self::Teleburn(teleburn) => teleburn.run(), Self::Traits(traits) => traits.run(), + Self::Transfer(transfer) => transfer.run(options), Self::Wallet(wallet) => wallet.run(options), } } diff --git a/src/subcommand/children.rs b/src/subcommand/children.rs new file mode 100644 index 0000000000..dc9f355ddf --- /dev/null +++ b/src/subcommand/children.rs @@ -0,0 +1,16 @@ +use super::*; + +#[derive(Debug, Parser)] +pub(crate) struct Children { +} + +impl Children { + pub(crate) fn run(self, options: Options) -> SubcommandResult { + let index = Index::open(&options)?; + index.update()?; + + index.get_children()?; + + Ok(Box::new(Empty {})) + } +} diff --git a/src/subcommand/preview.rs b/src/subcommand/preview.rs index 657aaee7df..d25ae0fbba 100644 --- a/src/subcommand/preview.rs +++ b/src/subcommand/preview.rs @@ -81,6 +81,8 @@ impl Preview { super::wallet::inscribe::Inscribe { batch: None, cbor_metadata: None, + utxo: Vec::new(), + coin_control: false, commit_fee_rate: None, destination: None, dry_run: false, diff --git a/src/subcommand/server.rs b/src/subcommand/server.rs index 5c665565c4..a755c68e93 100644 --- a/src/subcommand/server.rs +++ b/src/subcommand/server.rs @@ -22,7 +22,7 @@ use { headers::UserAgent, http::{header, HeaderMap, HeaderValue, StatusCode, Uri}, response::{IntoResponse, Redirect, Response}, - routing::get, + routing::{get, post}, Router, TypedHeader, }, axum_server::Handle, @@ -33,7 +33,8 @@ use { caches::DirCache, AcmeConfig, }, - std::{cmp::Ordering, str, sync::Arc}, + std::{cmp::Ordering, collections::HashMap, str, sync::Arc}, + tokio::time::sleep, tokio_stream::StreamExt, tower_http::{ compression::CompressionLayer, @@ -50,6 +51,18 @@ pub struct ServerConfig { pub is_json_api_enabled: bool, } +#[derive(Serialize)] +pub struct Outputs { + pub output: OutPoint, + pub details: OutputJson, +} + +#[derive(Serialize)] +pub struct Ranges { + pub output: OutPoint, + pub ranges: Vec<(u64, u64)>, +} + enum InscriptionQuery { Id(InscriptionId), Number(i64), @@ -95,6 +108,49 @@ struct Search { query: String, } +#[derive(Serialize)] +struct MyInscriptionJson { + number: i64, + id: InscriptionId, + parent: Option, + address: Option, + output_value: Option, + sat: Option, + content_length: Option, + content_type: String, + timestamp: u32, + genesis_height: u64, + genesis_fee: u64, + genesis_transaction: Txid, + location: String, + output: String, + offset: u64, + children: Vec, +} + +#[derive(Serialize)] +struct SatoshiJson { + number: u64, + decimal: String, + degree: String, + percentile: String, + name: String, + cycle: u64, + epoch: u64, + period: u64, + block: u64, + offset: u64, + rarity: Rarity, + // timestamp: i64, +} + +#[derive(Serialize)] +struct StatsJson { + highest_block_indexed: Option, + lowest_inscription_number: Option, + highest_inscription_number: Option, +} + #[derive(RustEmbed)] #[folder = "static"] struct StaticAssets; @@ -208,11 +264,21 @@ impl Server { ) .route("/inscriptions/:from", get(Self::inscriptions_from)) .route("/inscriptions/:from/:n", get(Self::inscriptions_from_n)) + .route( + "/inscriptions_json/:start", + get(Self::inscriptions_json_start), + ) + .route( + "/inscriptions_json/:start/:end", + get(Self::inscriptions_json_start_end), + ) .route("/install.sh", get(Self::install_script)) .route("/ordinal/:sat", get(Self::ordinal)) .route("/output/:output", get(Self::output)) + .route("/outputs", post(Self::outputs)) .route("/preview/:inscription_id", get(Self::preview)) .route("/range/:start/:end", get(Self::range)) + .route("/ranges", post(Self::ranges)) .route("/rare.txt", get(Self::rare_txt)) .route("/rune/:rune", get(Self::rune)) .route("/runes", get(Self::runes)) @@ -220,7 +286,11 @@ impl Server { .route("/search", get(Self::search_by_query)) .route("/search/*query", get(Self::search_by_path)) .route("/static/*path", get(Self::static_asset)) + .route("/stats", get(Self::stats)) .route("/status", get(Self::status)) + .route("/transfers/:height", get(Self::inscriptionids_from_height)) + .route("/transfers/:height/:start", get(Self::inscriptionids_from_height_start)) + .route("/transfers/:height/:start/:end", get(Self::inscriptionids_from_height_start_end)) .route("/tx/:txid", get(Self::transaction)) .layer(Extension(index)) .layer(Extension(page_config)) @@ -413,6 +483,7 @@ impl Server { } async fn clock(Extension(index): Extension>) -> ServerResult { + log::info!("GET /clock"); Ok( ( [( @@ -431,6 +502,7 @@ impl Server { Path(DeserializeFromStr(sat)): Path>, accept_json: AcceptJson, ) -> ServerResult { + log::info!("GET /sat/{sat}"); let inscriptions = index.get_inscription_ids_by_sat(sat)?; let satpoint = index.rare_sat_satpoint(sat)?.or_else(|| { inscriptions.first().and_then(|&first_inscription_id| { @@ -472,6 +544,7 @@ impl Server { } async fn ordinal(Path(sat): Path) -> Redirect { + log::info!("GET /ordinal/{sat}"); Redirect::to(&format!("/sat/{sat}")) } @@ -481,6 +554,7 @@ impl Server { Path(outpoint): Path, accept_json: AcceptJson, ) -> ServerResult { + log::info!("GET /output/{outpoint}"); let list = index.list(outpoint)?; let output = if outpoint == OutPoint::null() || outpoint == unbound_outpoint() { @@ -537,6 +611,80 @@ impl Server { }) } + async fn outputs( + Extension(page_config): Extension>, + Extension(index): Extension>, + Json(data): Json + ) -> ServerResult { + log::info!("POST /outputs"); + + if !data.is_array() { + return Err(ServerError::BadRequest("expected array".to_string())); + } + + let mut result = Vec::new(); + + for outpoint in data.as_array().unwrap() { + if !outpoint.is_string() { + return Err(ServerError::BadRequest("expected array of strings".to_string())); + } + + match OutPoint::from_str(outpoint.as_str().unwrap()) { + Ok(outpoint) => { + sleep(Duration::from_millis(0)).await; + + let list = index.list(outpoint)?; + + let output = if outpoint == OutPoint::null() || outpoint == unbound_outpoint() { + let mut value = 0; + + if let Some(List::Unspent(ranges)) = &list { + for (start, end) in ranges { + value += end - start; + } + } + + TxOut { + value, + script_pubkey: ScriptBuf::new(), + } + } else { + index + .get_transaction(outpoint.txid)? + .ok_or_not_found(|| format!("output {outpoint}"))? + .output + .into_iter() + .nth(outpoint.vout as usize) + .ok_or_not_found(|| format!("output {outpoint}"))? + }; + + let inscriptions = index.get_inscriptions_on_output(outpoint)?; + + let runes = index.get_rune_balances_for_outpoint(outpoint)?; + + result.push( + Outputs {output: outpoint, details: + OutputJson::new( + outpoint, + list, + page_config.chain, + output, + inscriptions, + runes + .into_iter() + .map(|(rune, pile)| (rune, pile.amount)) + .collect(), + ) + } + ) + } + _ => return Err(ServerError::BadRequest(format!("expected array of OutPoint strings ({} is bad)", outpoint))), + } + } + + Ok(Json(result).into_response()) + } + async fn range( Extension(page_config): Extension>, Path((DeserializeFromStr(start), DeserializeFromStr(end))): Path<( @@ -544,6 +692,7 @@ impl Server { DeserializeFromStr, )>, ) -> ServerResult> { + log::info!("GET /range/{start}/{end}"); match start.cmp(&end) { Ordering::Equal => Err(ServerError::BadRequest("empty range".to_string())), Ordering::Greater => Err(ServerError::BadRequest( @@ -553,7 +702,57 @@ impl Server { } } + async fn ranges( + Extension(index): Extension>, + Json(data): Json + ) -> ServerResult { + log::info!("POST /ranges"); + + if !index.has_sat_index() { + return Err(ServerError::BadRequest("the /ranges endpoint needs the server to have a sat index".to_string())); + } + + if !data.is_array() { + return Err(ServerError::BadRequest("expected array".to_string())); + } + + let mut result = Vec::new(); + let mut range_count = 0; + let mut outpoint_count = 0; + let start_time = Instant::now(); + + for outpoint in data.as_array().unwrap() { + if start_time.elapsed() > Duration::from_secs(5) { + return Err(ServerError::BadRequest("request timed out".to_string())); + } + + if !outpoint.is_string() { + return Err(ServerError::BadRequest("expected array of strings".to_string())); + } + + match OutPoint::from_str(outpoint.as_str().unwrap()) { + Ok(outpoint) => { + sleep(Duration::from_millis(0)).await; + match index.ranges(outpoint) { + Ok(ranges) => { + range_count += ranges.len(); + outpoint_count += 1; + result.push(Ranges {output: outpoint, ranges}); + } + _ => println!("no ranges for {}", outpoint), + } + } + _ => return Err(ServerError::BadRequest(format!("expected array of OutPoint strings ({} is bad)", outpoint))), + } + } + + println!(" {} ranges from {} outputs in {:?}", range_count, outpoint_count, start_time.elapsed()); + + Ok(Json(result).into_response()) + } + async fn rare_txt(Extension(index): Extension>) -> ServerResult { + log::info!("GET /rare.txt"); Ok(RareTxt(index.rare_sat_satpoints()?)) } @@ -603,6 +802,7 @@ impl Server { Extension(page_config): Extension>, Extension(index): Extension>, ) -> ServerResult> { + log::info!("GET /"); let blocks = index.blocks(100)?; let mut featured_blocks = BTreeMap::new(); for (height, hash) in blocks.iter().take(5) { @@ -616,6 +816,7 @@ impl Server { } async fn install_script() -> Redirect { + log::info!("GET /install.sh"); Redirect::to("https://raw.githubusercontent.com/ordinals/ord/master/install.sh") } @@ -626,6 +827,7 @@ impl Server { ) -> ServerResult> { let (block, height) = match query { BlockQuery::Height(height) => { + log::info!("GET /block/{height}/"); let block = index .get_block_by_height(height)? .ok_or_not_found(|| format!("block {height}"))?; @@ -633,6 +835,7 @@ impl Server { (block, height) } BlockQuery::Hash(hash) => { + log::info!("GET /block/{hash}/"); let info = index .block_header_info(hash)? .ok_or_not_found(|| format!("block {hash}"))?; @@ -660,11 +863,104 @@ impl Server { ) } + async fn inscriptionids_from_height( + Extension(page_config): Extension>, + Extension(index): Extension>, + Path(height): Path, + ) -> ServerResult { + log::info!("GET /transfers/{height}"); + Self::inscriptionids_from_height_inner(page_config, index.clone(), index.get_inscription_ids_by_height(height)?).await + } + + async fn inscriptionids_from_height_start( + Extension(page_config): Extension>, + Extension(index): Extension>, + Path(path): Path<(u64, usize)>, + ) -> ServerResult { + let height = path.0; + let start = path.1; + log::info!("GET /transfers/{height}/{start}"); + + let inscription_ids = index.get_inscription_ids_by_height(height)?; + let end = inscription_ids.len(); + + match start.cmp(&end) { + Ordering::Equal => Err(ServerError::BadRequest("range length == 0".to_string())), + Ordering::Greater => Err(ServerError::BadRequest("range length < 0".to_string())), + Ordering::Less => { + Self::inscriptionids_from_height_inner(page_config, index.clone(), inscription_ids[start..end].to_vec()).await + } + } + } + + async fn inscriptionids_from_height_start_end( + Extension(page_config): Extension>, + Extension(index): Extension>, + Path(path): Path<(u64, usize, usize)>, + ) -> ServerResult { + let height = path.0; + let start = path.1; + let mut end = path.2; + log::info!("GET /transfers/{height}/{start}/{end}"); + + let inscription_ids = index.get_inscription_ids_by_height(height)?; + end = usize::min(end, inscription_ids.len()); + + match start.cmp(&end) { + Ordering::Equal => Err(ServerError::BadRequest("range length == 0".to_string())), + Ordering::Greater => Err(ServerError::BadRequest("range length < 0".to_string())), + Ordering::Less => { + Self::inscriptionids_from_height_inner(page_config, index.clone(), inscription_ids[start..end].to_vec()).await + } + } + } + + async fn inscriptionids_from_height_inner( + page_config: Arc, + index: Arc, + inscription_ids: Vec, + ) -> ServerResult { + let mut ret = String::from(""); + let mut tx_cache = HashMap::new(); + for inscription_id in inscription_ids { + sleep(Duration::from_millis(0)).await; + let satpoint = index + .get_inscription_satpoint_by_id(inscription_id)? + .ok_or_not_found(|| format!("inscription {inscription_id}"))?; + let address = if satpoint.outpoint == unbound_outpoint() { + String::from("unbound") + } else { + if !tx_cache.contains_key(&satpoint.outpoint.txid) { + tx_cache.insert(satpoint.outpoint.txid, + index + .get_transaction(satpoint.outpoint.txid)? + .ok_or_not_found(|| format!("inscription {inscription_id} current transaction"))?); + } + + let output = tx_cache.get(&satpoint.outpoint.txid).unwrap().clone() + .output + .into_iter() + .nth(satpoint.outpoint.vout.try_into().unwrap()) + .ok_or_not_found(|| format!("inscription {inscription_id} current transaction output"))?; + if let Ok(address) = page_config.chain.address_from_script(&output.script_pubkey) { + address.to_string() + } else { + String::from("error") + } + }; + + ret += &format!("{} {}\n", inscription_id, address); + } + + Ok(ret) + } + async fn transaction( Extension(page_config): Extension>, Extension(index): Extension>, Path(txid): Path, ) -> ServerResult> { + log::info!("GET /tx/{txid}"); let inscription = index.get_inscription_by_id(InscriptionId { txid, index: 0 })?; let blockhash = index.get_transaction_blockhash(txid)?; @@ -682,7 +978,22 @@ impl Server { ) } + async fn stats(Extension(index): Extension>) -> ServerResult { + log::info!("GET /stats"); + let stats = index.get_stats()?; + Ok( + serde_json::to_string_pretty(&StatsJson { + highest_block_indexed: stats.0, + lowest_inscription_number: stats.1, + highest_inscription_number: stats.2, + }) + .ok() + .unwrap(), + ) + } + async fn status(Extension(index): Extension>) -> (StatusCode, &'static str) { + log::info!("GET /status"); if index.is_unrecoverably_reorged() { ( StatusCode::OK, @@ -700,6 +1011,7 @@ impl Server { Extension(index): Extension>, Query(search): Query, ) -> ServerResult { + log::info!("GET /search"); Self::search(&index, &search.query).await } @@ -707,6 +1019,7 @@ impl Server { Extension(index): Extension>, Path(search): Path, ) -> ServerResult { + log::info!("GET /search/{}", search.query); Self::search(&index, &search.query).await } @@ -751,6 +1064,7 @@ impl Server { } async fn favicon(user_agent: Option>) -> ServerResult { + log::info!("GET /favicon.ico"); if user_agent .map(|user_agent| { user_agent.as_str().contains("Safari/") @@ -782,6 +1096,7 @@ impl Server { Extension(page_config): Extension>, Extension(index): Extension>, ) -> ServerResult { + log::info!("GET /feed.xml"); let mut builder = rss::ChannelBuilder::default(); let chain = page_config.chain; @@ -822,8 +1137,10 @@ impl Server { async fn static_asset(Path(path): Path) -> ServerResult { let content = StaticAssets::get(if let Some(stripped) = path.strip_prefix('/') { + log::info!("GET /static/{stripped}"); stripped } else { + log::info!("GET /static/{path}"); &path }) .ok_or_not_found(|| format!("asset {path}"))?; @@ -838,10 +1155,12 @@ impl Server { } async fn block_count(Extension(index): Extension>) -> ServerResult { + log::info!("GET /blockcount"); Ok(index.block_count()?.to_string()) } async fn block_height(Extension(index): Extension>) -> ServerResult { + log::info!("GET /blockheight"); Ok( index .block_height()? @@ -851,6 +1170,7 @@ impl Server { } async fn block_hash(Extension(index): Extension>) -> ServerResult { + log::info!("GET /blockhash"); Ok( index .block_hash(None)? @@ -863,6 +1183,7 @@ impl Server { Extension(index): Extension>, Path(height): Path, ) -> ServerResult { + log::info!("GET /blockhash/{height}"); Ok( index .block_hash(Some(height))? @@ -872,6 +1193,7 @@ impl Server { } async fn block_time(Extension(index): Extension>) -> ServerResult { + log::info!("GET /blocktime"); Ok( index .block_time(index.block_height()?.ok_or_not_found(|| "blocktime")?)? @@ -885,6 +1207,7 @@ impl Server { Extension(index): Extension>, Path(path): Path<(u64, usize, usize)>, ) -> Result, ServerError> { + log::info!("GET /input/{}/{}/{}", path.0, path.1, path.2); let not_found = || format!("input /{}/{}/{}", path.0, path.1, path.2); let block = index @@ -907,10 +1230,12 @@ impl Server { } async fn faq() -> Redirect { + log::info!("GET /faq"); Redirect::to("https://docs.ordinals.com/faq/") } async fn bounties() -> Redirect { + log::info!("GET /bounties"); Redirect::to("https://docs.ordinals.com/bounty/") } @@ -919,6 +1244,7 @@ impl Server { Extension(config): Extension>, Path(inscription_id): Path, ) -> ServerResult { + log::info!("GET /content/{inscription_id}"); if config.is_hidden(inscription_id) { return Ok(PreviewUnknownHtml.into_response()); } @@ -965,6 +1291,7 @@ impl Server { Extension(config): Extension>, Path(inscription_id): Path, ) -> ServerResult { + log::info!("GET /preview/{inscription_id}"); if config.is_hidden(inscription_id) { return Ok(PreviewUnknownHtml.into_response()); } @@ -1054,10 +1381,16 @@ impl Server { accept_json: AcceptJson, ) -> ServerResult { let inscription_id = match query { - InscriptionQuery::Id(id) => id, - InscriptionQuery::Number(inscription_number) => index - .get_inscription_id_by_inscription_number(inscription_number)? - .ok_or_not_found(|| format!("{inscription_number}"))?, + InscriptionQuery::Id(id) => { + log::info!("GET /inscription/{id}"); + id + }, + InscriptionQuery::Number(inscription_number) => { + log::info!("GET /inscription/{inscription_number}"); + index + .get_inscription_id_by_inscription_number(inscription_number)? + .ok_or_not_found(|| format!("{inscription_number}"))? + }, }; let entry = index @@ -1141,6 +1474,7 @@ impl Server { Extension(index): Extension>, accept_json: AcceptJson, ) -> ServerResult { + log::info!("GET /inscriptions"); Self::inscriptions_inner(page_config, index, None, 100, accept_json).await } @@ -1150,6 +1484,7 @@ impl Server { Path(block_height): Path, accept_json: AcceptJson, ) -> ServerResult { + log::info!("GET /inscriptions/block/{block_height}"); Self::inscriptions_in_block_from_page( Extension(page_config), Extension(index), @@ -1165,6 +1500,7 @@ impl Server { Path((block_height, page_index)): Path<(u64, usize)>, accept_json: AcceptJson, ) -> ServerResult { + log::info!("GET /inscriptions/block/{block_height}/{page_index}"); let inscriptions = index.get_inscriptions_in_block(block_height)?; Ok(if accept_json.0 { @@ -1187,6 +1523,7 @@ impl Server { Path(from): Path, accept_json: AcceptJson, ) -> ServerResult { + log::info!("GET /inscriptions/{from}"); Self::inscriptions_inner(page_config, index, Some(from), 100, accept_json).await } @@ -1196,6 +1533,7 @@ impl Server { Path((from, n)): Path<(u64, usize)>, accept_json: AcceptJson, ) -> ServerResult { + log::info!("GET /inscriptions/{from}/{n}"); Self::inscriptions_inner(page_config, index, Some(from), n, accept_json).await } @@ -1228,6 +1566,151 @@ impl Server { }) } + async fn inscriptions_json_start( + Extension(page_config): Extension>, + Extension(index): Extension>, + Path(start): Path, + ) -> ServerResult { + log::info!("GET /inscriptions_json/{start}"); + Self::inscriptions_json(page_config, index, start, start + 1).await + } + + async fn inscriptions_json_start_end( + Extension(page_config): Extension>, + Extension(index): Extension>, + Path(path): Path<(i64, i64)>, + ) -> ServerResult { + log::info!("GET /inscriptions_json/{}/{}", path.0, path.1); + Self::inscriptions_json(page_config, index, path.0, path.1).await + } + + async fn inscriptions_json( + page_config: Arc, + index: Arc, + start: i64, + end: i64, + ) -> ServerResult { + const MAX_JSON_INSCRIPTIONS: i64 = 1000; + + match start.cmp(&end) { + Ordering::Equal => Err(ServerError::BadRequest("range length == 0".to_string())), + Ordering::Greater => Err(ServerError::BadRequest("range length < 0".to_string())), + Ordering::Less => { + if end - start > MAX_JSON_INSCRIPTIONS { + return Err(ServerError::BadRequest(format!( + "range length > {MAX_JSON_INSCRIPTIONS}" + ))); + } + + let mut ret = Vec::new(); + + for i in start..end { + sleep(Duration::from_millis(0)).await; + match index.get_inscription_id_by_inscription_number(i) { + Err(_) => return Err(ServerError::BadRequest(format!("no inscription {i}"))), + Ok(inscription_id) => match inscription_id { + Some(inscription_id) => { + let entry = index + .get_inscription_entry(inscription_id)? + .ok_or_not_found(|| format!("inscription {inscription_id}"))?; + + let tx = index.get_transaction(inscription_id.txid)?.unwrap(); + let inscription = index + .get_inscription_by_id(inscription_id)? + .ok_or_not_found(|| format!("inscription {inscription_id}"))?; + + let satpoint = index + .get_inscription_satpoint_by_id(inscription_id)? + .ok_or_not_found(|| format!("inscription {inscription_id}"))?; + + let output = if satpoint.outpoint == unbound_outpoint() { + None + } else { + Some( + if satpoint.outpoint.txid == inscription_id.txid { + tx + } else { + index + .get_transaction(satpoint.outpoint.txid)? + .ok_or_not_found(|| { + format!("inscription {inscription_id} current transaction") + })? + } + .output + .into_iter() + .nth(satpoint.outpoint.vout.try_into().unwrap()) + .ok_or_not_found(|| { + format!("inscription {inscription_id} current transaction output") + })?, + ) + }; + + let mut address = None; + if let Some(output) = &output { + if let Ok(a) = page_config.chain.address_from_script(&output.script_pubkey) { + address = Some(a.to_string()); + } + } + + let sat = entry.sat.map(|s| SatoshiJson { + number: s.n(), + decimal: s.decimal().to_string(), + degree: s.degree().to_string(), + percentile: s.percentile().to_string(), + name: s.name(), + cycle: s.cycle(), + epoch: s.epoch().0, + period: s.period(), + block: s.height().0, + offset: s.third(), + rarity: s.rarity(), + // timestamp: index.block_time(s.height())?.unix_timestamp(), + }); + + let content_type = inscription.content_type(); + let unbound_suffix = if satpoint.outpoint == unbound_outpoint() { + " (unbound)" + } else { + "" + }; + + ret.push(MyInscriptionJson { + number: i, + id: inscription_id, + parent: entry.parent, + address, + output_value: if output.is_some() { + Some(output.unwrap().value) + } else { + None + }, + sat, + content_length: inscription.content_length(), + content_type: if content_type.is_some() { + content_type.unwrap().to_string() + } else { + "".to_string() + }, + timestamp: entry.timestamp, + genesis_height: entry.height, + genesis_fee: entry.fee, + genesis_transaction: inscription_id.txid, + location: satpoint.to_string() + unbound_suffix, + output: satpoint.outpoint.to_string() + unbound_suffix, + offset: satpoint.offset, + children: index.get_children_by_inscription_id(inscription_id)?, + }); + } + None => return Err(ServerError::BadRequest(format!("no inscription {i}"))), + }, + } + } + + Ok(serde_json::to_string_pretty(&ret).ok().unwrap()) + } + } + } + async fn redirect_http_to_https( Extension(mut destination): Extension, uri: Uri, diff --git a/src/subcommand/transfer.rs b/src/subcommand/transfer.rs new file mode 100644 index 0000000000..7d7adf10e7 --- /dev/null +++ b/src/subcommand/transfer.rs @@ -0,0 +1,45 @@ +use super::*; + +#[derive(Debug, Parser)] +pub(crate) struct Transfer { + #[clap(long, help = "Delete the whole transfer log table.")] + delete: bool, + #[clap(long, help = "Delete transfer logs for blocks before height .")] + trim: Option, +} + +impl Transfer { + pub(crate) fn run(self, options: Options) -> SubcommandResult { + let index = Index::open(&options)?; + index.update()?; + + if self.delete && self.trim.is_some() { + return Err(anyhow!("Cannot use both --delete and --trim")); + } + + if self.delete { + println!("deleting transfer log table"); + index.delete_transfer_log()?; + return Ok(Box::new(Empty {})); + } + + if self.trim.is_some() { + let trim = self.trim.unwrap(); + println!("deleting transfer logs for blocks before {trim}"); + index.trim_transfer_log(trim)?; + } + + let (rows, first_key, last_key) = index.show_transfer_log_stats()?; + if rows == 0 { + println!("the transfer table has {rows} rows"); + } else { + println!( + "the transfer table has {rows} rows from height {} to height {}", + first_key.unwrap(), + last_key.unwrap() + ); + } + + Ok(Box::new(Empty {})) + } +} diff --git a/src/subcommand/wallet/inscribe.rs b/src/subcommand/wallet/inscribe.rs index a0f9c641a4..0cd76107e6 100644 --- a/src/subcommand/wallet/inscribe.rs +++ b/src/subcommand/wallet/inscribe.rs @@ -60,6 +60,13 @@ pub(crate) struct Inscribe { conflicts_with = "json_metadata" )] pub(crate) cbor_metadata: Option, + #[arg( + long, + help = "Consider spending outpoint , even if it is unconfirmed or contains inscriptions" + )] + pub(crate) utxo: Vec, + #[arg(long, help = "Only spend outpoints given with --utxo")] + pub(crate) coin_control: bool, #[arg( long, help = "Use sats/vbyte for commit transaction.\nDefaults to if unset." @@ -108,10 +115,23 @@ impl Inscribe { let index = Index::open(&options)?; index.update()?; - let utxos = index.get_unspent_outputs(Wallet::load(&options)?)?; + let mut utxos = if self.coin_control { + BTreeMap::new() + } else { + index.get_unspent_outputs(Wallet::load(&options)?)? + }; let client = options.bitcoin_rpc_client_for_wallet_command(false)?; + for outpoint in &self.utxo { + utxos.insert( + *outpoint, + Amount::from_sat( + client.get_raw_transaction(&outpoint.txid, None)?.output[outpoint.vout as usize].value, + ), + ); + } + let chain = options.chain(); let postage = self.postage.unwrap_or(TransactionBuilder::TARGET_POSTAGE); diff --git a/src/subcommand/wallet/send.rs b/src/subcommand/wallet/send.rs index 1ad4af17b4..9d8fadaf20 100644 --- a/src/subcommand/wallet/send.rs +++ b/src/subcommand/wallet/send.rs @@ -4,6 +4,16 @@ use {super::*, crate::subcommand::wallet::transaction_builder::Target, crate::wa pub(crate) struct Send { address: Address, outgoing: Outgoing, + #[arg( + long, + help = "Consider spending outpoint , even if it is unconfirmed or contains inscriptions" + )] + utxo: Vec, + #[clap( + long, + help = "Only spend outpoints given with --utxo when sending inscriptions or satpoints" + )] + pub(crate) coin_control: bool, #[arg(long, help = "Use fee rate of sats/vB")] fee_rate: FeeRate, #[arg( @@ -32,7 +42,20 @@ impl Send { let client = options.bitcoin_rpc_client_for_wallet_command(false)?; - let unspent_outputs = index.get_unspent_outputs(Wallet::load(&options)?)?; + let mut unspent_outputs = if self.coin_control { + BTreeMap::new() + } else { + index.get_unspent_outputs(Wallet::load(&options)?)? + }; + + for outpoint in &self.utxo { + unspent_outputs.insert( + *outpoint, + Amount::from_sat( + client.get_raw_transaction(&outpoint.txid, None)?.output[outpoint.vout as usize].value, + ), + ); + } let inscriptions = index.get_inscriptions(&unspent_outputs)?; @@ -49,6 +72,9 @@ impl Send { .get_inscription_satpoint_by_id(id)? .ok_or_else(|| anyhow!("Inscription {id} not found"))?, Outgoing::Amount(amount) => { + if self.coin_control || !self.utxo.is_empty() { + bail!("--coin_control and --utxo don't work when sending cardinals"); + } Self::lock_inscriptions(&client, inscriptions, unspent_outputs)?; let txid = Self::send_amount(&client, amount, address, self.fee_rate.n())?; return Ok(Box::new(Output { transaction: txid }));