{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://catalog.lintel.tools/schemas/schemastore/eidolon-resource/_shared/latest--SentenceTransformersTokenTextSplitter.json",
  "title": "SentenceTransformersTokenTextSplitter",
  "x-lintel": {
    "source": "https://www.eidolonai.com/json_schema/v1/schemas/DocumentTransformer/SentenceTransformersTokenTextSplitter.json",
    "sourceSha256": "c963c8c348a002f776d1b19c78485246db83514b503fae488ce608ad84aff1b5"
  },
  "type": "object",
  "properties": {
    "implementation": {
      "const": "SentenceTransformersTokenTextSplitter",
      "title": "Implementation"
    },
    "chunk_size": {
      "default": 4000,
      "description": "Maximum size of chunks to return",
      "title": "Chunk Size",
      "type": "integer"
    },
    "chunk_overlap": {
      "default": 50,
      "title": "Chunk Overlap",
      "type": "integer"
    },
    "keep_separator": {
      "default": false,
      "description": "Whether to keep the separator in the chunks",
      "title": "Keep Separator",
      "type": "boolean"
    },
    "strip_whitespace": {
      "default": true,
      "description": "If `True`, strips whitespace from the start and end of every document",
      "title": "Strip Whitespace",
      "type": "boolean"
    },
    "model": {
      "default": "sentence-transformers/all-mpnet-base-v2",
      "description": "Model name",
      "title": "Model",
      "type": "string"
    },
    "tokens_per_chunk": {
      "anyOf": [
        {
          "type": "integer"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "description": "Number of tokens per chunk",
      "title": "Tokens Per Chunk"
    }
  },
  "additionalProperties": false,
  "required": [
    "implementation"
  ],
  "reference_details": {
    "clz": "eidolon_ai_sdk.agent.doc_manager.transformer.text_splitters.SentenceTransformersTokenTextSplitter",
    "groups": [
      "DocumentTransformer"
    ],
    "name": "SentenceTransformersTokenTextSplitter",
    "overrides": {}
  }
}