Skip to content

Commit 81e41b3

Browse files
blythedblythed
committed
Add remote template support and patch api-key
1 parent a0c8c20 commit 81e41b3

File tree

8 files changed

+88
-39
lines changed

8 files changed

+88
-39
lines changed

‎.github/workflows/ci_templates.yaml

-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ on:
44
pull_request:
55
branches:
66
- main
7-
- '[0-9]+.[0-9]+'
87
paths: # Paths that may affect code quality
98

109
concurrency:

‎CHANGELOG.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
**Before you create a Pull Request, remember to update the Changelog with your changes.**
99

10-
## Changes Since Last Release
10+
## Changes Since Last Release
1111

1212
#### Changed defaults / behaviours
1313

‎superduper/cli/main.py

+18-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import json
22
import os
3-
import subprocess
43

54
from superduper import CFG, Component, logging, superduper
65
from superduper.components.template import Template
@@ -103,9 +102,20 @@ def bootstrap(
103102

104103
db = superduper(data_backend)
105104
existing = db.show('template')
105+
106+
if template.startswith('http'):
107+
import subprocess
108+
109+
logging.info('Downloading remote template...')
110+
subprocess.run(['curl', '-O', '-k', template])
111+
template = template.split('/')[-1]
112+
106113
if destination is not None:
107-
root = os.path.dirname(os.path.dirname(__file__))
108-
template_directory = os.path.join(root, f'templates/{template}')
114+
if os.path.exists(template):
115+
template_directory = template
116+
else:
117+
root = os.path.dirname(os.path.dirname(__file__))
118+
template_directory = os.path.join(root, f'templates/{template}')
109119
print(template_directory)
110120
import shutil
111121

@@ -114,8 +124,12 @@ def bootstrap(
114124

115125
if template in existing:
116126
logging.warn(f'Template {template} already exists')
127+
117128
logging.info(f'Applying template: {template} from inbuilt')
118-
tem = getattr(inbuilt, template)
129+
if os.path.exists(template):
130+
tem = Template.read(template)
131+
else:
132+
tem = getattr(inbuilt, template)
119133
if tem.requirements and pip_install:
120134
with open('/tmp/requirements.txt', 'w') as f:
121135
f.write('\n'.join(tem.requirements))

‎templates/simple_rag/VERSION

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0.5.0.dev
1+
0.4.0

‎templates/simple_rag/blobs/a03a6ac18d448d0e3cd7b62ebcd7f19d777b5a59

11.1 KB
Binary file not shown.

‎templates/simple_rag/blobs/de5a10b374e634d964148beb865f73c5b82e53a6

766 Bytes
Binary file not shown.

‎templates/simple_rag/build.ipynb

+58-23
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
},
4343
{
4444
"cell_type": "code",
45-
"execution_count": null,
45+
"execution_count": 1,
4646
"id": "3ef70f6d-a189-460a-8864-241a689624e2",
4747
"metadata": {
4848
"editable": true,
@@ -66,7 +66,7 @@
6666
},
6767
{
6868
"cell_type": "code",
69-
"execution_count": null,
69+
"execution_count": 2,
7070
"id": "cb029a5e-fedf-4f07-8a31-d220cfbfbb3d",
7171
"metadata": {
7272
"editable": true,
@@ -75,7 +75,22 @@
7575
},
7676
"tags": []
7777
},
78-
"outputs": [],
78+
"outputs": [
79+
{
80+
"name": "stdout",
81+
"output_type": "stream",
82+
"text": [
83+
"2024-Nov-11 11:49:18.44| INFO | superduper.misc.plugins:13 | Loading plugin: mongodb\n",
84+
"2024-Nov-11 11:49:18.49| INFO | superduper.base.datalayer:76 | Building Data Layer\n",
85+
"2024-Nov-11 11:49:18.49| INFO | superduper.base.build:184 | Configuration: \n",
86+
" +---------------+--------------+\n",
87+
"| Configuration | Value |\n",
88+
"+---------------+--------------+\n",
89+
"| Data Backend | mongomock:// |\n",
90+
"+---------------+--------------+\n"
91+
]
92+
}
93+
],
7994
"source": [
8095
"from superduper import superduper, CFG\n",
8196
"\n",
@@ -88,7 +103,7 @@
88103
},
89104
{
90105
"cell_type": "code",
91-
"execution_count": null,
106+
"execution_count": 3,
92107
"id": "4e7902bd",
93108
"metadata": {
94109
"editable": true,
@@ -114,7 +129,7 @@
114129
},
115130
{
116131
"cell_type": "code",
117-
"execution_count": null,
132+
"execution_count": 4,
118133
"id": "1ef8dd07-1b47-4dce-84dd-a081d1f5ee9d",
119134
"metadata": {},
120135
"outputs": [],
@@ -136,7 +151,7 @@
136151
},
137152
{
138153
"cell_type": "code",
139-
"execution_count": null,
154+
"execution_count": 5,
140155
"id": "c5965fdf",
141156
"metadata": {},
142157
"outputs": [],
@@ -170,7 +185,7 @@
170185
},
171186
{
172187
"cell_type": "code",
173-
"execution_count": null,
188+
"execution_count": 6,
174189
"id": "2d20eaa0-a416-4483-938e-23f79845739a",
175190
"metadata": {},
176191
"outputs": [],
@@ -198,7 +213,7 @@
198213
},
199214
{
200215
"cell_type": "code",
201-
"execution_count": null,
216+
"execution_count": 7,
202217
"id": "93d21872-d4dc-40dc-abab-fb07ba102ea3",
203218
"metadata": {},
204219
"outputs": [],
@@ -217,7 +232,7 @@
217232
},
218233
{
219234
"cell_type": "code",
220-
"execution_count": null,
235+
"execution_count": 8,
221236
"id": "31900eec-b516-4bef-939e-2e8f46252b12",
222237
"metadata": {},
223238
"outputs": [],
@@ -265,7 +280,7 @@
265280
},
266281
{
267282
"cell_type": "code",
268-
"execution_count": null,
283+
"execution_count": 9,
269284
"id": "a9b1f538-65ca-499e-b6d0-2dd733f81723",
270285
"metadata": {},
271286
"outputs": [],
@@ -276,7 +291,8 @@
276291
"from superduper_openai import OpenAIEmbedding\n",
277292
"\n",
278293
"openai_embedding = OpenAIEmbedding(\n",
279-
" identifier='text-embedding-ada-002',\n",
294+
" identifier='text-embedding',\n",
295+
" model='text-embedding-ada-002',\n",
280296
" datatype=sqlvector(shape=(1536,)),\n",
281297
" client_kwargs={'base_url': BASE_URL, 'api_key': API_KEY},\n",
282298
")"
@@ -292,7 +308,7 @@
292308
},
293309
{
294310
"cell_type": "code",
295-
"execution_count": null,
311+
"execution_count": 10,
296312
"id": "4663fa4b-c2ec-427d-bf8b-b8b109cc2ccf",
297313
"metadata": {},
298314
"outputs": [],
@@ -315,7 +331,7 @@
315331
},
316332
{
317333
"cell_type": "code",
318-
"execution_count": null,
334+
"execution_count": 11,
319335
"id": "509c3505-54c5-4e68-84ec-3df8bea0fd74",
320336
"metadata": {},
321337
"outputs": [],
@@ -335,7 +351,7 @@
335351
},
336352
{
337353
"cell_type": "code",
338-
"execution_count": null,
354+
"execution_count": 12,
339355
"id": "f98e5ff4",
340356
"metadata": {},
341357
"outputs": [],
@@ -360,7 +376,7 @@
360376
},
361377
{
362378
"cell_type": "code",
363-
"execution_count": null,
379+
"execution_count": 13,
364380
"id": "44baeb09-6f35-4cf2-b814-46283a59f7e9",
365381
"metadata": {},
366382
"outputs": [],
@@ -387,7 +403,7 @@
387403
},
388404
{
389405
"cell_type": "code",
390-
"execution_count": null,
406+
"execution_count": 14,
391407
"id": "2d3a0d3a-da1c-41ec-b16c-f281c46ad794",
392408
"metadata": {},
393409
"outputs": [],
@@ -398,7 +414,7 @@
398414
},
399415
{
400416
"cell_type": "code",
401-
"execution_count": null,
417+
"execution_count": 15,
402418
"id": "2a82ea22-9694-4c65-b72f-c89ae49d1ab2",
403419
"metadata": {},
404420
"outputs": [],
@@ -417,7 +433,7 @@
417433
},
418434
{
419435
"cell_type": "code",
420-
"execution_count": null,
436+
"execution_count": 16,
421437
"id": "e6787c78-4b14-4a72-818b-450408a74331",
422438
"metadata": {},
423439
"outputs": [],
@@ -436,7 +452,7 @@
436452
},
437453
{
438454
"cell_type": "code",
439-
"execution_count": null,
455+
"execution_count": 17,
440456
"id": "e7c16557-af76-4e70-83d9-2984e19a9554",
441457
"metadata": {},
442458
"outputs": [],
@@ -463,10 +479,21 @@
463479
},
464480
{
465481
"cell_type": "code",
466-
"execution_count": null,
482+
"execution_count": 18,
467483
"id": "2e850c03-33c6-4c88-95d3-d14146a6a0af",
468484
"metadata": {},
469-
"outputs": [],
485+
"outputs": [
486+
{
487+
"name": "stdout",
488+
"output_type": "stream",
489+
"text": [
490+
"2024-Nov-11 11:49:19.42| WARNING | superduper.base.document:479 | Leaf listener:chunker already exists\n",
491+
"2024-Nov-11 11:49:19.42| WARNING | superduper.base.document:479 | Leaf model:chunker already exists\n",
492+
"2024-Nov-11 11:49:19.43| WARNING | superduper.base.document:479 | Leaf datatype:dill already exists\n",
493+
"2024-Nov-11 11:49:19.43| WARNING | superduper.base.document:479 | Leaf var-table-name-select-var-id-field-x already exists\n"
494+
]
495+
}
496+
],
470497
"source": [
471498
"from superduper import Template, Table, Schema\n",
472499
"from superduper.components.dataset import RemoteData\n",
@@ -526,10 +553,18 @@
526553
},
527554
{
528555
"cell_type": "code",
529-
"execution_count": null,
556+
"execution_count": 19,
530557
"id": "8924ba0d-7c01-4d6c-87fb-245531db7506",
531558
"metadata": {},
532-
"outputs": [],
559+
"outputs": [
560+
{
561+
"name": "stdout",
562+
"output_type": "stream",
563+
"text": [
564+
"2024-Nov-11 11:49:19.43| WARNING | superduper.base.document:479 | Leaf str already exists\n"
565+
]
566+
}
567+
],
533568
"source": [
534569
"template.export('.')"
535570
]

‎templates/simple_rag/component.json

+10-9
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,19 @@
2020
"method": "dill",
2121
"encodable": "artifact"
2222
},
23-
"a395902bca4bf34255e97a6a057b0edffc3500b2": {
23+
"de5a10b374e634d964148beb865f73c5b82e53a6": {
2424
"_path": "superduper.components.datatype.Artifact",
2525
"datatype": "?datatype:dill",
2626
"uri": null,
27-
"blob": "&:blob:a395902bca4bf34255e97a6a057b0edffc3500b2"
27+
"blob": "&:blob:de5a10b374e634d964148beb865f73c5b82e53a6"
2828
},
2929
"dataset:superduper-docs": {
3030
"_path": "superduper.components.dataset.RemoteData",
3131
"upstream": null,
3232
"plugins": null,
3333
"cache": true,
3434
"status": null,
35-
"getter": "?a395902bca4bf34255e97a6a057b0edffc3500b2"
35+
"getter": "?de5a10b374e634d964148beb865f73c5b82e53a6"
3636
},
3737
"table:sample_simple_rag": {
3838
"_path": "superduper.components.table.Table",
@@ -58,14 +58,14 @@
5858
"method": "dill",
5959
"encodable": "artifact"
6060
},
61-
"727d3bb560939e1211f9cac189d56e07e9622eeb": {
61+
"a03a6ac18d448d0e3cd7b62ebcd7f19d777b5a59": {
6262
"_path": "superduper.components.datatype.Artifact",
6363
"datatype": "?datatype:dill",
6464
"uri": null,
65-
"blob": "&:blob:727d3bb560939e1211f9cac189d56e07e9622eeb"
65+
"blob": "&:blob:a03a6ac18d448d0e3cd7b62ebcd7f19d777b5a59"
6666
},
6767
"model:chunker": {
68-
"_object": "?727d3bb560939e1211f9cac189d56e07e9622eeb",
68+
"_object": "?a03a6ac18d448d0e3cd7b62ebcd7f19d777b5a59",
6969
"upstream": null,
7070
"plugins": null,
7171
"cache": true,
@@ -108,7 +108,7 @@
108108
1536
109109
]
110110
},
111-
"model:<var:embedding_model>": {
111+
"model:text-embedding": {
112112
"_path": "superduper_openai.model.OpenAIEmbedding",
113113
"upstream": null,
114114
"plugins": null,
@@ -155,7 +155,7 @@
155155
"status": null,
156156
"cdc_table": "<var:output_prefix>chunker__?(listener:chunker.uuid)",
157157
"key": "<var:output_prefix>chunker__?(listener:chunker.uuid)",
158-
"model": "?model:<var:embedding_model>",
158+
"model": "?model:text-embedding",
159159
"predict_kwargs": {},
160160
"select": "?outputs-chunker-?(listener:chunker.uuid)-select-id-source-outputs-chunker-?(listener:chunker.uuid)",
161161
"flatten": false
@@ -293,6 +293,7 @@
293293
"default": null
294294
}
295295
},
296+
"schema": null,
296297
"blobs": null,
297298
"files": null,
298299
"requirements": null,
@@ -304,4 +305,4 @@
304305
}
305306
},
306307
"_files": {}
307-
}
308+
}

0 commit comments

Comments
 (0)