partition data with : "part": { "pos": "ra, dec", "num-stripes": 340, "num-sub-stripes": 3, "chunk": "chunkId", "sub-chunk": "subChunkId", "overlap": 0.01667 } sph-partition --config-file=wp311a.json --in.path=gaia_wise_test_matches.csv --mr.num-workers=6 --mr.pool-size=32768 --mr.block-size=16 --verbose --out.dir=chunks/ create database : curl http://localhost:8080/ingest/database -X POST -H "Content-Type: application/json" -d '{"database":"testWP311a","auto_build_secondary_index":1,"num_stripes":340,"num_sub_stripes":3,"overlap":0.01667,"auth_key":""}' {"database":{"chunk_id_key":"","database":"testWP311a","director_table":"","family_name":"layout_340_3","is_published":0,"sub_chunk_id_key":"","tables":{}},"error":"","error_ext":{},"success":1} create Xmatch table : cat xmatcha.json | curl http://localhost:8080/ingest/table -X POST -H "Content-Type: application/json" -d@- {"database":{"chunk_id_key":"chunkId","columns":{"Xmatch":[{"name":"qserv_trans_id","type":"INT NOT NULL"},{"name":"gaia_designation","type":"VARCHAR(32)"},{"name":"ra","type":"DOUBLE NOT NULL"},{"name":"dec","type":"DOUBLE NOT NULL"},{"name":"gaia_bp","type":"DOUBLE"},{"name":"gaia_g","type":"DOUBLE"},{"name":"gaia_rp","type":"DOUBLE"},{"name":"wise_designation","type":"VARCHAR(32)"},{"name":"wise_ra","type":"DOUBLE"},{"name":"wise_dec","type":"DOUBLE"},{"name":"wise_w1","type":"DOUBLE"},{"name":"wise_w2","type":"DOUBLE"},{"name":"wise_w3","type":"DOUBLE"},{"name":"wise_w4","type":"DOUBLE"},{"name":"match_p","type":"DOUBLE"},{"name":"separation","type":"DOUBLE"},{"name":"eta","type":"DOUBLE"},{"name":"xi","type":"DOUBLE"},{"name":"gaia_avg_cont","type":"DOUBLE"},{"name":"wise_avg_cont","type":"DOUBLE"},{"name":"gaia_cont_p1","type":"DOUBLE"},{"name":"gaia_cont_p10","type":"DOUBLE"},{"name":"wise_cont_p1","type":"DOUBLE"},{"name":"wise_cont_p10","type":"DOUBLE"},{"name":"objectId","type":"BIGINT NOT NULL"},{"name":"wise_cntr","type":"BIGINT"},{"name":"chunkId","type":"INT UNSIGNED NOT NULL"},{"name":"subChunkId","type":"INT UNSIGNED NOT NULL"}]},"database":"testWP311a","director_table":"Xmatch","family_name":"layout_340_3","is_published":0,"sub_chunk_id_key":"subChunkId","tables":{"Xmatch":{"director_key":"objectId","is_partitioned":1,"latitude_key":"dec","longitude_key":"ra","name":"Xmatch"}}},"error":"","error_ext":{},"success":1} locate one chunk : curl http://localhost:8080/ingest/chunk -X POST -H "Content-Type: application/json" -d '{"database":"testWP311a","chunk":179619,"auth_key":""}' {"error":"","error_ext":{},"location":{"host":"qserv-worker-0.qserv-worker","http_host":"qserv-worker-0.qserv-worker","http_port":25004,"port":25002,"worker":"qserv-worker-0"},"success":1} get transaction : curl http://localhost:8080/ingest/trans -X POST -H "Content-Type: application/json" -d '{"database":"testWP311a","auth_key":""}' {"databases":{"testWP311a":{"num_chunks":1,"transactions":[{"begin_time":1636029401740,"context":{},"database":"testWP311a","end_time":0,"id":13,"state":"STARTED"}]}},"error":"","error_ext":{},"success":1} load the chunk : source /qserv/stack/loadLSST.bash setup qserv_distrib -t qserv-dev qserv-replica-file INGEST FILE-LIST XmatchLoada.json --verbose commit : curl http://localhost:8080/ingest/trans/13?abort=0 -X PUT -H "Content-Type: application/json" -d '{"auth_key":""}' {"databases":{"testWP311a":{"num_chunks":1,"transactions":[{"begin_time":1636029401740,"context":{},"database":"testWP311a","end_time":1636029592874,"id":13,"state":"FINISHED"}]}},"error":"","error_ext":{},"secondary-index-build-success":0,"success":1} publish : curl http://localhost:8080/ingest/database/testWP311a -X PUT -H "Content-Type: application/json" -d '{"auth_key":""}' {"database":{"chunk_id_key":"chunkId","columns":{"Xmatch":[{"name":"qserv_trans_id","type":"INT NOT NULL"},{"name":"gaia_designation","type":"VARCHAR(32)"},{"name":"ra","type":"DOUBLE NOT NULL"},{"name":"dec","type":"DOUBLE NOT NULL"},{"name":"gaia_bp","type":"DOUBLE"},{"name":"gaia_g","type":"DOUBLE"},{"name":"gaia_rp","type":"DOUBLE"},{"name":"wise_designation","type":"VARCHAR(32)"},{"name":"wise_ra","type":"DOUBLE"},{"name":"wise_dec","type":"DOUBLE"},{"name":"wise_w1","type":"DOUBLE"},{"name":"wise_w2","type":"DOUBLE"},{"name":"wise_w3","type":"DOUBLE"},{"name":"wise_w4","type":"DOUBLE"},{"name":"match_p","type":"DOUBLE"},{"name":"separation","type":"DOUBLE"},{"name":"eta","type":"DOUBLE"},{"name":"xi","type":"DOUBLE"},{"name":"gaia_avg_cont","type":"DOUBLE"},{"name":"wise_avg_cont","type":"DOUBLE"},{"name":"gaia_cont_p1","type":"DOUBLE"},{"name":"gaia_cont_p10","type":"DOUBLE"},{"name":"wise_cont_p1","type":"DOUBLE"},{"name":"wise_cont_p10","type":"DOUBLE"},{"name":"objectId","type":"BIGINT NOT NULL"},{"name":"wise_cntr","type":"BIGINT"},{"name":"chunkId","type":"INT UNSIGNED NOT NULL"},{"name":"subChunkId","type":"INT UNSIGNED NOT NULL"}]},"database":"testWP311a","director_table":"Xmatch","family_name":"layout_340_3","is_published":1,"sub_chunk_id_key":"subChunkId","tables":{"Xmatch":{"director_key":"objectId","is_partitioned":1,"latitude_key":"dec","longitude_key":"ra","name":"Xmatch"}}},"error":"","error_ext":{},"success":1}