refactor duckdb to use internal function for writing to file

Author: rfhb

DESCRIPTION

@@ -16,7 +16,9 @@ Authors@R: c(
            role = "aut",
            comment = c(ORCID = "0000-0003-1444-9135")),
     person("Rich", "FitzJohn", role = "aut"),
-    person("Jeroen", "Ooms", role = "aut")
+    person("Jeroen", "Ooms", role = "aut"),
+    person("Ivan", "Tarbakou", role = "cph", 
+           comment = "mongo-to-sql-converter library")
   )
 License: MIT + file LICENSE
 LazyData: true

NEWS.md

@@ -1,6 +1,7 @@
 # nodbi 0.11.0.9000
 
-## Bug fixes
+## Improvements
+* Use `duckdb` internal function for writing NDJSON to file
 * Removing message about RSQLite handling NDJSON file name as value
 
 # nodbi 0.11.0

R/get.R

@@ -34,7 +34,7 @@ docdb_get <- function(src, key, limit = NULL, ...) {
   if (length(params[["fields"]]) ||
       length(params[["query"]]) ||
       length(params[["listfields"]])) stop(
-    "Use docdb_query() to specify fields or query parameters.")
+        "Use docdb_query() to specify fields or query parameters.")
 
   # dispatch
   UseMethod("docdb_get", src)
@@ -167,29 +167,52 @@ sqlGet <- function(src, key, limit = NULL, getFunction, ...) {
     # canonical sorting in nodbi
     "ORDER BY _id ASC;")
 
-  # temporary file for streaming
-  tfname <- tempfile()
-  tfnameCon <- file(description = tfname, open = "wt")
-  # register to remove file after used for streaming
-  on.exit(try(close(tfnameCon), silent = TRUE), add = TRUE)
-  on.exit(try(unlink(tfname), silent = TRUE), add = TRUE)
-
-  # get data, write to file in ndjson format
-  writeLines(
-    stringi::stri_replace_all_fixed(
-      str = paste0(
-        "", # protect against empty query result
-        stats::na.omit(  # eliminate rows without json
-          DBI::dbGetQuery(
-            conn = src$con,
-            statement = statement,
-            n = n)[["json"]])),
-      pattern = "\n",
-      replacement = "\\n"),
-    con = tfnameCon,
-    sep = "\n",
-    useBytes = TRUE)
-  close(tfnameCon)
+  # use duckdb internal function
+  if (inherits(src, "src_duckdb")) {
+
+    # temporary file for streaming
+    tfname <- tempfile()
+    on.exit(try(unlink(tfname), silent = TRUE), add = TRUE)
+
+    # modify statement to export as file
+    statement <- paste0(
+      "COPY (", sub(";$", "", statement),
+      ifelse(n == -1L, "", paste0(" LIMIT ", n)),
+      ") TO '", tfname, "' (HEADER false, QUOTE '');"
+    )
+
+    # write ndjson
+    DBI::dbExecute(
+      conn = src$con,
+      statement = statement)
+
+  } else {
+
+    # temporary file for streaming
+    tfname <- tempfile()
+    tfnameCon <- file(description = tfname, open = "wt")
+    # register to remove file after used for streaming
+    on.exit(try(close(tfnameCon), silent = TRUE), add = TRUE)
+    on.exit(try(unlink(tfname), silent = TRUE), add = TRUE)
+
+    # get data, write to file in ndjson format
+    writeLines(
+      stringi::stri_replace_all_fixed(
+        str = paste0(
+          "", # protect against empty query result
+          stats::na.omit(  # eliminate rows without json
+            DBI::dbGetQuery(
+              conn = src$con,
+              statement = statement,
+              n = n)[["json"]])),
+        pattern = "\n",
+        replacement = "\\n"),
+      con = tfnameCon,
+      sep = "\n",
+      useBytes = TRUE)
+    close(tfnameCon)
+
+  }
 
   # stream in ndjson records
   return(jsonlite::stream_in(file(tfname), verbose = FALSE))

R/query.R

@@ -1464,14 +1464,43 @@ processDbGetQuery <- function(
             eval.parent(parse(text = getData))),
           verbose = FALSE))
 
-  # temporary file and connection
-  tfname <- tempfile()
-  on.exit(try(unlink(tfname), silent = TRUE), add = TRUE)
+  # TODO
+
+  # use duckdb internal function
+  if (inherits(eval.parent(parse(text = "src")), "src_duckdb")) {
+
+    # get data
+    statement <- eval.parent(parse(text = "statement"))
+    n <- eval.parent(parse(text = "n"))
+
+    # temporary file for streaming
+    tfname <- tempfile()
+    on.exit(try(unlink(tfname), silent = TRUE), add = TRUE)
+
+    # modify statement to export as file
+    statement <- paste0(
+      "COPY (", sub(";$", "", statement),
+      ifelse(n == -1L, "", paste0(" LIMIT ", n)),
+      ") TO '", tfname, "' (HEADER false, QUOTE '');"
+    )
+
+    # write ndjson
+    DBI::dbExecute(
+      conn = eval.parent(parse(text = "src"))$con,
+      statement = statement)
+
+  } else {
 
-  # write out
-  writeLines(
-    eval.parent(parse(text = getData)),
-    con = tfname)
+    # temporary file and connection
+    tfname <- tempfile()
+    on.exit(try(unlink(tfname), silent = TRUE), add = TRUE)
+
+    # write out
+    writeLines(
+      eval.parent(parse(text = getData)),
+      con = tfname)
+
+  }
 
   # early exit
   if (file.size(tfname) <= 2L) return(NULL)

README.Rmd

@@ -325,14 +325,14 @@ result <- rbenchmark::benchmark(
 )
 
 # 2025-03-29 with M3 hardware, databases via homebrew
-result[rev(order(result$elapsed)), c("test", "replications", "elapsed")]
+# result[rev(order(result$elapsed)), c("test", "replications", "elapsed")]
 #         test replications elapsed
-# 4    CouchDB            3   69.69
-# 3    Elastic            3   26.79
-# 1    MongoDB            3    1.61
+# 4    CouchDB            3   61.22
+# 3    Elastic            3   27.79
 # 5 PostgreSQL            3    1.52
-# 6     DuckDB            3    1.12
-# 2     SQLite            3    0.67
+# 1    MongoDB            3    1.51
+# 6     DuckDB            3    1.01
+# 2     SQLite            3    0.65
 
 message(R.version$version.string)
 # R Under development (unstable) (2025-03-10 r87922)

README.md

@@ -358,14 +358,14 @@ result <- rbenchmark::benchmark(
 )
 
 # 2025-03-29 with M3 hardware, databases via homebrew
-result[rev(order(result$elapsed)), c("test", "replications", "elapsed")]
+# result[rev(order(result$elapsed)), c("test", "replications", "elapsed")]
 #         test replications elapsed
-# 4    CouchDB            3   69.69
-# 3    Elastic            3   26.79
-# 1    MongoDB            3    1.61
+# 4    CouchDB            3   61.22
+# 3    Elastic            3   27.79
 # 5 PostgreSQL            3    1.52
-# 6     DuckDB            3    1.12
-# 2     SQLite            3    0.67
+# 1    MongoDB            3    1.51
+# 6     DuckDB            3    1.01
+# 2     SQLite            3    0.65
 
 message(R.version$version.string)
 # R Under development (unstable) (2025-03-10 r87922)

codemeta.json

@@ -46,6 +46,13 @@
       "familyName": "Ooms"
     }
   ],
+  "copyrightHolder": [
+    {
+      "@type": "Person",
+      "givenName": "Ivan",
+      "familyName": "Tarbakou"
+    }
+  ],
   "maintainer": [
     {
       "@type": "Person",
@@ -309,7 +316,7 @@
   "applicationCategory": "Databases",
   "isPartOf": "https://ropensci.org",
   "keywords": ["database", "MongoDB", "Elasticsearch", "CouchDB", "SQLite", "PostgreSQL", "DuckDB", "NoSQL", "JSON", "documents", "rstats", "nosql", "couchdb", "mongodb", "elasticsearch", "r", "r-package", "sqlite", "postgresql"],
-  "fileSize": "665.808KB",
+  "fileSize": "667.226KB",
   "releaseNotes": "https://github.com/ropensci/nodbi/blob/master/NEWS.md",
   "readme": "https://github.com/ropensci/nodbi/blob/master/README.md",
   "contIntegration": ["https://github.com/ropensci/nodbi/actions?query=workflow%3AR-CMD-check", "https://app.codecov.io/gh/rfhb/nodbi"],

docs/articles/nodbi-overview.html

@@ -3,7 +3,7 @@ pkgdown: 2.1.1
 pkgdown_sha: ~
 articles:
   nodbi-overview: nodbi-overview.html
-last_built: 2025-03-29T16:07Z
+last_built: 2025-03-29T18:05Z
 urls:
   reference: https://docs.ropensci.org/nodbi/reference
   article: https://docs.ropensci.org/nodbi/articles

docs/authors.html

@@ -7,6 +7,7 @@
 # git diff > ../update.patch
 # rm -rf mongo-to-sql-converter
 #
+# Ivan Tarbakou, MIT License
 module="mongo-to-sql-converter"
 cd inst/js
 npm install $module