PD12MThis is a curated PD12M dataset for use with the II-Commons project.
This dataset comprises a curated Public Domain 12M image collection, refined by filtering for active image links. EXIF data was extracted, and images underwent preprocessing and feature extraction using SigLIP 2. All vector embeddings are normalized 16-bit half-precision vectors optimized for L2 indexing with vectorchord.
This dataset is derived and organized from Spawning/PD12M. The original license information for the image can be found in the corresponding entry of the original database.
PostgreSQL 17 with extensions: vectorchord and pg_search
The easiest way is to use our Docker image, or build your own. Then load the psql_basebackup branch, following the Quick Start
Ensure extensions are enabled, connect to the database using the psql, and run the following SQL:
CREATE EXTENSION IF NOT EXISTS vchord CASCADE;
CREATE EXTENSION IF NOT EXISTS pg_search CASCADE;
This dataset is available for a wide range of applications.
Here is a demo of how to use the dataset with II-Commons.
CREATE TABLE IF NOT EXISTS is_pd12m (
id BIGSERIAL PRIMARY KEY,
url VARCHAR NOT NULL,
caption VARCHAR NOT NULL DEFAULT '',
caption_long VARCHAR DEFAULT '',
origin_width BIGINT NOT NULL DEFAULT 0,
origin_height BIGINT NOT NULL DEFAULT 0,
processed_width BIGINT NOT NULL DEFAULT 0,
processed_height BIGINT NOT NULL DEFAULT 0,
aspect_ratio DOUBLE PRECISION NOT NULL DEFAULT 0,
exif JSONB NOT NULL DEFAULT '{}',
meta JSONB NOT NULL DEFAULT '{}',
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
source JSONB NOT NULL DEFAULT '[]',
vector halfvec(1152) DEFAULT NULL,
origin_source VARCHAR DEFAULT '',
license VARCHAR DEFAULT ''
);
\copy is_pd12m FROM 'data/0000000.csv' CSV HEADER;
\copy is_pd12m FROM 'data/0000001.csv' CSV HEADER;
\copy is_pd12m FROM 'data/0000002.csv' CSV HEADER;
...
copy is_pd12m FROM 'data/0000000.csv' CSV HEADER;
copy is_pd12m FROM 'data/0000001.csv' CSV HEADER;
copy is_pd12m FROM 'data/0000002.csv' CSV HEADER;
...
You need to create the following indexes for the best performance.
The vector column is a halfvec(1152) column, which is a 16-bit half-precision vector optimized for L2 indexing with vectorchord. You can get more information about the vector index from the vectorchord documentation.
CREATE UNIQUE INDEX IF NOT EXISTS is_pd12m_url_index ON is_pd12m (url);
CREATE INDEX IF NOT EXISTS is_pd12m_origin_width_index ON is_pd12m (origin_width);
CREATE INDEX IF NOT EXISTS is_pd12m_origin_height_index ON is_pd12m (origin_height);
CREATE INDEX IF NOT EXISTS is_pd12m_processed_width_index ON is_pd12m (processed_width);
CREATE INDEX IF NOT EXISTS is_pd12m_processed_height_index ON is_pd12m (processed_height);
CREATE INDEX IF NOT EXISTS is_pd12m_aspect_ratio_index ON is_pd12m (aspect_ratio);
CREATE INDEX IF NOT EXISTS is_pd12m_exif_index ON is_pd12m USING gin(exif);
CREATE INDEX IF NOT EXISTS is_pd12m_meta_index ON is_pd12m USING gin(meta);
CREATE INDEX IF NOT EXISTS is_pd12m_source_index ON is_pd12m USING gin(source);
CREATE INDEX IF NOT EXISTS is_pd12m_created_at_index ON is_pd12m (created_at);
CREATE INDEX IF NOT EXISTS is_pd12m_updated_at_index ON is_pd12m (updated_at);
CREATE INDEX IF NOT EXISTS is_pd12m_vector_index ON is_pd12m USING vchordrq (vector halfvec_l2_ops) WITH (options = $$
residual_quantization = true
[build.internal]
lists = [20000]
build_threads = 6
spherical_centroids = false
$$);
CREATE INDEX IF NOT EXISTS is_pd12m_caption_index ON is_pd12m (caption) WHERE caption = '';
CREATE INDEX IF NOT EXISTS is_pd12m_caption_long_index ON is_pd12m (caption_long) WHERE caption_long = '';
CREATE INDEX IF NOT EXISTS is_pd12m_vector_null_index ON is_pd12m (vector) WHERE vector IS NULL;
Click this link to learn how to query the dataset with II-Commons.
PD12MThis is a curated PD12M dataset for use with the II-Commons project.
This dataset comprises a curated Public Domain 12M image collection, refined by filtering for active image links. EXIF data was extracted, and images underwent preprocessing and feature extraction using SigLIP 2. All vector embeddings are normalized 16-bit half-precision vectors optimized for L2 indexing with vectorchord.
This dataset is derived and organized from Spawning/PD12M. The original license information for the image can be found in the corresponding entry of the original database.
PostgreSQL 17 with extensions: vectorchord and pg_search
The easiest way is to use our Docker image, or build your own. Then load the psql_basebackup branch, following the Quick Start
Ensure extensions are enabled, connect to the database using the psql, and run the following SQL:
CREATE EXTENSION IF NOT EXISTS vchord CASCADE;
CREATE EXTENSION IF NOT EXISTS pg_search CASCADE;
This dataset is available for a wide range of applications.
Here is a demo of how to use the dataset with II-Commons.
CREATE TABLE IF NOT EXISTS is_pd12m (
id BIGSERIAL PRIMARY KEY,
url VARCHAR NOT NULL,
caption VARCHAR NOT NULL DEFAULT '',
caption_long VARCHAR DEFAULT '',
origin_width BIGINT NOT NULL DEFAULT 0,
origin_height BIGINT NOT NULL DEFAULT 0,
processed_width BIGINT NOT NULL DEFAULT 0,
processed_height BIGINT NOT NULL DEFAULT 0,
aspect_ratio DOUBLE PRECISION NOT NULL DEFAULT 0,
exif JSONB NOT NULL DEFAULT '{}',
meta JSONB NOT NULL DEFAULT '{}',
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
source JSONB NOT NULL DEFAULT '[]',
vector halfvec(1152) DEFAULT NULL,
origin_source VARCHAR DEFAULT '',
license VARCHAR DEFAULT ''
);
\copy is_pd12m FROM 'data/0000000.csv' CSV HEADER;
\copy is_pd12m FROM 'data/0000001.csv' CSV HEADER;
\copy is_pd12m FROM 'data/0000002.csv' CSV HEADER;
...
copy is_pd12m FROM 'data/0000000.csv' CSV HEADER;
copy is_pd12m FROM 'data/0000001.csv' CSV HEADER;
copy is_pd12m FROM 'data/0000002.csv' CSV HEADER;
...
You need to create the following indexes for the best performance.
The vector column is a halfvec(1152) column, which is a 16-bit half-precision vector optimized for L2 indexing with vectorchord. You can get more information about the vector index from the vectorchord documentation.
CREATE UNIQUE INDEX IF NOT EXISTS is_pd12m_url_index ON is_pd12m (url);
CREATE INDEX IF NOT EXISTS is_pd12m_origin_width_index ON is_pd12m (origin_width);
CREATE INDEX IF NOT EXISTS is_pd12m_origin_height_index ON is_pd12m (origin_height);
CREATE INDEX IF NOT EXISTS is_pd12m_processed_width_index ON is_pd12m (processed_width);
CREATE INDEX IF NOT EXISTS is_pd12m_processed_height_index ON is_pd12m (processed_height);
CREATE INDEX IF NOT EXISTS is_pd12m_aspect_ratio_index ON is_pd12m (aspect_ratio);
CREATE INDEX IF NOT EXISTS is_pd12m_exif_index ON is_pd12m USING gin(exif);
CREATE INDEX IF NOT EXISTS is_pd12m_meta_index ON is_pd12m USING gin(meta);
CREATE INDEX IF NOT EXISTS is_pd12m_source_index ON is_pd12m USING gin(source);
CREATE INDEX IF NOT EXISTS is_pd12m_created_at_index ON is_pd12m (created_at);
CREATE INDEX IF NOT EXISTS is_pd12m_updated_at_index ON is_pd12m (updated_at);
CREATE INDEX IF NOT EXISTS is_pd12m_vector_index ON is_pd12m USING vchordrq (vector halfvec_l2_ops) WITH (options = $$
residual_quantization = true
[build.internal]
lists = [20000]
build_threads = 6
spherical_centroids = false
$$);
CREATE INDEX IF NOT EXISTS is_pd12m_caption_index ON is_pd12m (caption) WHERE caption = '';
CREATE INDEX IF NOT EXISTS is_pd12m_caption_long_index ON is_pd12m (caption_long) WHERE caption_long = '';
CREATE INDEX IF NOT EXISTS is_pd12m_vector_null_index ON is_pd12m (vector) WHERE vector IS NULL;
Click this link to learn how to query the dataset with II-Commons.