【新增】AI 知识库：文档向量化 demo
This commit is contained in:
xiaoxin
parent
cb59a61a04
commit
2a984504d9
@ -0,0 +1,16 @@
|
||||
package cn.iocoder.yudao.module.ai.service.knowledge;
|
||||
|
||||
/**
|
||||
* AI 知识库 Service 接口
|
||||
*
|
||||
* @author xiaoxin
|
||||
*/
|
||||
public interface DocService {
|
||||
|
||||
|
||||
/**
|
||||
* 向量化文档
|
||||
*/
|
||||
void embeddingDoc();
|
||||
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
package cn.iocoder.yudao.module.ai.service.knowledge;
|
||||
|
||||
import jakarta.annotation.Resource;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.reader.TextReader;
|
||||
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
|
||||
import org.springframework.ai.vectorstore.RedisVectorStore;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* AI 知识库 Service 实现类
|
||||
*
|
||||
* @author xiaoxin
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class DocServiceImpl implements DocService {
|
||||
|
||||
@Resource
|
||||
RedisVectorStore vectorStore;
|
||||
@Resource
|
||||
TokenTextSplitter tokenTextSplitter;
|
||||
|
||||
// TODO @xin 临时测试用，后续删
|
||||
@Value("classpath:/webapp/test/Fel.pdf")
|
||||
private org.springframework.core.io.Resource data;
|
||||
|
||||
|
||||
@Override
|
||||
public void embeddingDoc() {
|
||||
// 读取文件
|
||||
org.springframework.core.io.Resource file = data;
|
||||
TextReader loader = new TextReader(file);
|
||||
List<Document> documents = loader.get();
|
||||
// 文档分段
|
||||
List<Document> segments = tokenTextSplitter.apply(documents);
|
||||
// 向量化并存储
|
||||
vectorStore.add(segments);
|
||||
}
|
||||
}
|
||||
@ -39,6 +39,22 @@
|
||||
<artifactId>spring-ai-stability-ai-spring-boot-starter</artifactId>
|
||||
<version>${spring-ai.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-transformers-spring-boot-starter</artifactId>
|
||||
<version>${spring-ai.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.ai</groupId>
|
||||
<artifactId>spring-ai-redis-store</artifactId>
|
||||
<version>${spring-ai.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.data</groupId>
|
||||
<artifactId>spring-data-redis</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>cn.iocoder.boot</groupId>
|
||||
|
||||
@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright 2023 - 2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.ai.autoconfigure.vectorstore.redis;
|
||||
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.vectorstore.RedisVectorStore;
|
||||
import org.springframework.ai.vectorstore.RedisVectorStore.RedisVectorStoreConfig;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
|
||||
import redis.clients.jedis.JedisPooled;
|
||||
|
||||
/**
|
||||
* TODO @xin 先拿 spring-ai 最新代码覆盖，1.0.0-M1 跟 redis 自动配置会冲突
|
||||
*
|
||||
* @author Christian Tzolov
|
||||
* @author Eddú Meléndez
|
||||
*/
|
||||
@AutoConfiguration(after = RedisAutoConfiguration.class)
|
||||
@ConditionalOnClass({JedisPooled.class, JedisConnectionFactory.class, RedisVectorStore.class, EmbeddingModel.class})
|
||||
//@ConditionalOnBean(JedisConnectionFactory.class)
|
||||
@EnableConfigurationProperties(RedisVectorStoreProperties.class)
|
||||
public class RedisVectorStoreAutoConfiguration {
|
||||
|
||||
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public RedisVectorStore vectorStore(EmbeddingModel embeddingModel, RedisVectorStoreProperties properties,
|
||||
JedisConnectionFactory jedisConnectionFactory) {
|
||||
|
||||
var config = RedisVectorStoreConfig.builder()
|
||||
.withIndexName(properties.getIndex())
|
||||
.withPrefix(properties.getPrefix())
|
||||
.build();
|
||||
|
||||
return new RedisVectorStore(config, embeddingModel,
|
||||
new JedisPooled(jedisConnectionFactory.getHostName(), jedisConnectionFactory.getPort()),
|
||||
properties.isInitializeSchema());
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,456 @@
|
||||
/*
|
||||
* Copyright 2023 - 2024 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.springframework.ai.vectorstore;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.ai.vectorstore.filter.FilterExpressionConverter;
|
||||
import org.springframework.beans.factory.InitializingBean;
|
||||
import org.springframework.util.Assert;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import redis.clients.jedis.JedisPooled;
|
||||
import redis.clients.jedis.Pipeline;
|
||||
import redis.clients.jedis.json.Path2;
|
||||
import redis.clients.jedis.search.*;
|
||||
import redis.clients.jedis.search.Schema.FieldType;
|
||||
import redis.clients.jedis.search.schemafields.*;
|
||||
import redis.clients.jedis.search.schemafields.VectorField.VectorAlgorithm;
|
||||
|
||||
import java.text.MessageFormat;
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* The RedisVectorStore is for managing and querying vector data in a Redis database. It
|
||||
* offers functionalities like adding, deleting, and performing similarity searches on
|
||||
* documents.
|
||||
*
|
||||
* The store utilizes RedisJSON and RedisSearch to handle JSON documents and to index and
|
||||
* search vector data. It supports various vector algorithms (e.g., FLAT, HSNW) for
|
||||
* efficient similarity searches. Additionally, it allows for custom metadata fields in
|
||||
* the documents to be stored alongside the vector and content data.
|
||||
*
|
||||
* This class requires a RedisVectorStoreConfig configuration object for initialization,
|
||||
* which includes settings like Redis URI, index name, field names, and vector algorithms.
|
||||
* It also requires an EmbeddingModel to convert documents into embeddings before storing
|
||||
* them.
|
||||
*
|
||||
* @author Julien Ruaux
|
||||
* @author Christian Tzolov
|
||||
* @author Eddú Meléndez
|
||||
* @see VectorStore
|
||||
* @see RedisVectorStoreConfig
|
||||
* @see EmbeddingModel
|
||||
*/
|
||||
public class RedisVectorStore implements VectorStore, InitializingBean {
|
||||
|
||||
public enum Algorithm {
|
||||
|
||||
FLAT, HSNW
|
||||
|
||||
}
|
||||
|
||||
public record MetadataField(String name, FieldType fieldType) {
|
||||
|
||||
public static MetadataField text(String name) {
|
||||
return new MetadataField(name, FieldType.TEXT);
|
||||
}
|
||||
|
||||
public static MetadataField numeric(String name) {
|
||||
return new MetadataField(name, FieldType.NUMERIC);
|
||||
}
|
||||
|
||||
public static MetadataField tag(String name) {
|
||||
return new MetadataField(name, FieldType.TAG);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration for the Redis vector store.
|
||||
*/
|
||||
public static final class RedisVectorStoreConfig {
|
||||
|
||||
private final String indexName;
|
||||
|
||||
private final String prefix;
|
||||
|
||||
private final String contentFieldName;
|
||||
|
||||
private final String embeddingFieldName;
|
||||
|
||||
private final Algorithm vectorAlgorithm;
|
||||
|
||||
private final List<MetadataField> metadataFields;
|
||||
|
||||
private RedisVectorStoreConfig() {
|
||||
this(builder());
|
||||
}
|
||||
|
||||
private RedisVectorStoreConfig(Builder builder) {
|
||||
this.indexName = builder.indexName;
|
||||
this.prefix = builder.prefix;
|
||||
this.contentFieldName = builder.contentFieldName;
|
||||
this.embeddingFieldName = builder.embeddingFieldName;
|
||||
this.vectorAlgorithm = builder.vectorAlgorithm;
|
||||
this.metadataFields = builder.metadataFields;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start building a new configuration.
|
||||
* @return The entry point for creating a new configuration.
|
||||
*/
|
||||
public static Builder builder() {
|
||||
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
/**
|
||||
* {@return the default config}
|
||||
*/
|
||||
public static RedisVectorStoreConfig defaultConfig() {
|
||||
|
||||
return builder().build();
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
|
||||
private String indexName = DEFAULT_INDEX_NAME;
|
||||
|
||||
private String prefix = DEFAULT_PREFIX;
|
||||
|
||||
private String contentFieldName = DEFAULT_CONTENT_FIELD_NAME;
|
||||
|
||||
private String embeddingFieldName = DEFAULT_EMBEDDING_FIELD_NAME;
|
||||
|
||||
private Algorithm vectorAlgorithm = DEFAULT_VECTOR_ALGORITHM;
|
||||
|
||||
private List<MetadataField> metadataFields = new ArrayList<>();
|
||||
|
||||
private Builder() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Configures the Redis index name to use.
|
||||
* @param name the index name to use
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder withIndexName(String name) {
|
||||
this.indexName = name;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configures the Redis key prefix to use (default: "embedding:").
|
||||
* @param prefix the prefix to use
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder withPrefix(String prefix) {
|
||||
this.prefix = prefix;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configures the Redis content field name to use.
|
||||
* @param name the content field name to use
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder withContentFieldName(String name) {
|
||||
this.contentFieldName = name;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configures the Redis embedding field name to use.
|
||||
* @param name the embedding field name to use
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder withEmbeddingFieldName(String name) {
|
||||
this.embeddingFieldName = name;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configures the Redis vector algorithmto use.
|
||||
* @param algorithm the vector algorithm to use
|
||||
* @return this builder
|
||||
*/
|
||||
public Builder withVectorAlgorithm(Algorithm algorithm) {
|
||||
this.vectorAlgorithm = algorithm;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withMetadataFields(MetadataField... fields) {
|
||||
return withMetadataFields(Arrays.asList(fields));
|
||||
}
|
||||
|
||||
public Builder withMetadataFields(List<MetadataField> fields) {
|
||||
this.metadataFields = fields;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* {@return the immutable configuration}
|
||||
*/
|
||||
public RedisVectorStoreConfig build() {
|
||||
|
||||
return new RedisVectorStoreConfig(this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private final boolean initializeSchema;
|
||||
|
||||
public static final String DEFAULT_INDEX_NAME = "spring-ai-index";
|
||||
|
||||
public static final String DEFAULT_CONTENT_FIELD_NAME = "content";
|
||||
|
||||
public static final String DEFAULT_EMBEDDING_FIELD_NAME = "embedding";
|
||||
|
||||
public static final String DEFAULT_PREFIX = "embedding:";
|
||||
|
||||
public static final Algorithm DEFAULT_VECTOR_ALGORITHM = Algorithm.HSNW;
|
||||
|
||||
private static final String QUERY_FORMAT = "%s=>[KNN %s @%s $%s AS %s]";
|
||||
|
||||
private static final Path2 JSON_SET_PATH = Path2.of("$");
|
||||
|
||||
private static final String JSON_PATH_PREFIX = "$.";
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(RedisVectorStore.class);
|
||||
|
||||
private static final Predicate<Object> RESPONSE_OK = Predicate.isEqual("OK");
|
||||
|
||||
private static final Predicate<Object> RESPONSE_DEL_OK = Predicate.isEqual(1l);
|
||||
|
||||
private static final String VECTOR_TYPE_FLOAT32 = "FLOAT32";
|
||||
|
||||
private static final String EMBEDDING_PARAM_NAME = "BLOB";
|
||||
|
||||
public static final String DISTANCE_FIELD_NAME = "vector_score";
|
||||
|
||||
private static final String DEFAULT_DISTANCE_METRIC = "COSINE";
|
||||
|
||||
private final JedisPooled jedis;
|
||||
|
||||
private final EmbeddingModel embeddingModel;
|
||||
|
||||
private final RedisVectorStoreConfig config;
|
||||
|
||||
private FilterExpressionConverter filterExpressionConverter;
|
||||
|
||||
public RedisVectorStore(RedisVectorStoreConfig config, EmbeddingModel embeddingModel, JedisPooled jedis,
|
||||
boolean initializeSchema) {
|
||||
|
||||
Assert.notNull(config, "Config must not be null");
|
||||
Assert.notNull(embeddingModel, "Embedding model must not be null");
|
||||
this.initializeSchema = initializeSchema;
|
||||
|
||||
this.jedis = jedis;
|
||||
this.embeddingModel = embeddingModel;
|
||||
this.config = config;
|
||||
this.filterExpressionConverter = new RedisFilterExpressionConverter(this.config.metadataFields);
|
||||
}
|
||||
|
||||
public JedisPooled getJedis() {
|
||||
return this.jedis;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void add(List<Document> documents) {
|
||||
try (Pipeline pipeline = this.jedis.pipelined()) {
|
||||
for (Document document : documents) {
|
||||
var embedding = this.embeddingModel.embed(document);
|
||||
document.setEmbedding(embedding);
|
||||
|
||||
var fields = new HashMap<String, Object>();
|
||||
fields.put(this.config.embeddingFieldName, embedding);
|
||||
fields.put(this.config.contentFieldName, document.getContent());
|
||||
fields.putAll(document.getMetadata());
|
||||
pipeline.jsonSetWithEscape(key(document.getId()), JSON_SET_PATH, fields);
|
||||
}
|
||||
List<Object> responses = pipeline.syncAndReturnAll();
|
||||
Optional<Object> errResponse = responses.stream().filter(Predicate.not(RESPONSE_OK)).findAny();
|
||||
if (errResponse.isPresent()) {
|
||||
String message = MessageFormat.format("Could not add document: {0}", errResponse.get());
|
||||
if (logger.isErrorEnabled()) {
|
||||
logger.error(message);
|
||||
}
|
||||
throw new RuntimeException(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String key(String id) {
|
||||
return this.config.prefix + id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<Boolean> delete(List<String> idList) {
|
||||
try (Pipeline pipeline = this.jedis.pipelined()) {
|
||||
for (String id : idList) {
|
||||
pipeline.jsonDel(key(id));
|
||||
}
|
||||
List<Object> responses = pipeline.syncAndReturnAll();
|
||||
Optional<Object> errResponse = responses.stream().filter(Predicate.not(RESPONSE_DEL_OK)).findAny();
|
||||
if (errResponse.isPresent()) {
|
||||
if (logger.isErrorEnabled()) {
|
||||
logger.error("Could not delete document: {}", errResponse.get());
|
||||
}
|
||||
return Optional.of(false);
|
||||
}
|
||||
return Optional.of(true);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Document> similaritySearch(SearchRequest request) {
|
||||
|
||||
Assert.isTrue(request.getTopK() > 0, "The number of documents to returned must be greater than zero");
|
||||
Assert.isTrue(request.getSimilarityThreshold() >= 0 && request.getSimilarityThreshold() <= 1,
|
||||
"The similarity score is bounded between 0 and 1; least to most similar respectively.");
|
||||
|
||||
String filter = nativeExpressionFilter(request);
|
||||
|
||||
String queryString = String.format(QUERY_FORMAT, filter, request.getTopK(), this.config.embeddingFieldName,
|
||||
EMBEDDING_PARAM_NAME, DISTANCE_FIELD_NAME);
|
||||
|
||||
List<String> returnFields = new ArrayList<>();
|
||||
this.config.metadataFields.stream().map(MetadataField::name).forEach(returnFields::add);
|
||||
returnFields.add(this.config.embeddingFieldName);
|
||||
returnFields.add(this.config.contentFieldName);
|
||||
returnFields.add(DISTANCE_FIELD_NAME);
|
||||
var embedding = toFloatArray(this.embeddingModel.embed(request.getQuery()));
|
||||
Query query = new Query(queryString).addParam(EMBEDDING_PARAM_NAME, RediSearchUtil.toByteArray(embedding))
|
||||
.returnFields(returnFields.toArray(new String[0]))
|
||||
.setSortBy(DISTANCE_FIELD_NAME, true)
|
||||
.dialect(2);
|
||||
|
||||
SearchResult result = this.jedis.ftSearch(this.config.indexName, query);
|
||||
return result.getDocuments()
|
||||
.stream()
|
||||
.filter(d -> similarityScore(d) >= request.getSimilarityThreshold())
|
||||
.map(this::toDocument)
|
||||
.toList();
|
||||
}
|
||||
|
||||
private Document toDocument(redis.clients.jedis.search.Document doc) {
|
||||
var id = doc.getId().substring(this.config.prefix.length());
|
||||
var content = doc.hasProperty(this.config.contentFieldName) ? doc.getString(this.config.contentFieldName)
|
||||
: null;
|
||||
Map<String, Object> metadata = this.config.metadataFields.stream()
|
||||
.map(MetadataField::name)
|
||||
.filter(doc::hasProperty)
|
||||
.collect(Collectors.toMap(Function.identity(), doc::getString));
|
||||
metadata.put(DISTANCE_FIELD_NAME, 1 - similarityScore(doc));
|
||||
return new Document(id, content, metadata);
|
||||
}
|
||||
|
||||
private float similarityScore(redis.clients.jedis.search.Document doc) {
|
||||
return (2 - Float.parseFloat(doc.getString(DISTANCE_FIELD_NAME))) / 2;
|
||||
}
|
||||
|
||||
private String nativeExpressionFilter(SearchRequest request) {
|
||||
if (request.getFilterExpression() == null) {
|
||||
return "*";
|
||||
}
|
||||
return "(" + this.filterExpressionConverter.convertExpression(request.getFilterExpression()) + ")";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterPropertiesSet() {
|
||||
|
||||
if (!this.initializeSchema) {
|
||||
return;
|
||||
}
|
||||
|
||||
// If index already exists don't do anything
|
||||
if (this.jedis.ftList().contains(this.config.indexName)) {
|
||||
return;
|
||||
}
|
||||
|
||||
String response = this.jedis.ftCreate(this.config.indexName,
|
||||
FTCreateParams.createParams().on(IndexDataType.JSON).addPrefix(this.config.prefix), schemaFields());
|
||||
if (!RESPONSE_OK.test(response)) {
|
||||
String message = MessageFormat.format("Could not create index: {0}", response);
|
||||
throw new RuntimeException(message);
|
||||
}
|
||||
}
|
||||
|
||||
private Iterable<SchemaField> schemaFields() {
|
||||
Map<String, Object> vectorAttrs = new HashMap<>();
|
||||
vectorAttrs.put("DIM", this.embeddingModel.dimensions());
|
||||
vectorAttrs.put("DISTANCE_METRIC", DEFAULT_DISTANCE_METRIC);
|
||||
vectorAttrs.put("TYPE", VECTOR_TYPE_FLOAT32);
|
||||
List<SchemaField> fields = new ArrayList<>();
|
||||
fields.add(TextField.of(jsonPath(this.config.contentFieldName)).as(this.config.contentFieldName).weight(1.0));
|
||||
fields.add(VectorField.builder()
|
||||
.fieldName(jsonPath(this.config.embeddingFieldName))
|
||||
.algorithm(vectorAlgorithm())
|
||||
.attributes(vectorAttrs)
|
||||
.as(this.config.embeddingFieldName)
|
||||
.build());
|
||||
|
||||
if (!CollectionUtils.isEmpty(this.config.metadataFields)) {
|
||||
for (MetadataField field : this.config.metadataFields) {
|
||||
fields.add(schemaField(field));
|
||||
}
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
private SchemaField schemaField(MetadataField field) {
|
||||
String fieldName = jsonPath(field.name);
|
||||
switch (field.fieldType) {
|
||||
case NUMERIC:
|
||||
return NumericField.of(fieldName).as(field.name);
|
||||
case TAG:
|
||||
return TagField.of(fieldName).as(field.name);
|
||||
case TEXT:
|
||||
return TextField.of(fieldName).as(field.name);
|
||||
default:
|
||||
throw new IllegalArgumentException(
|
||||
MessageFormat.format("Field {0} has unsupported type {1}", field.name, field.fieldType));
|
||||
}
|
||||
}
|
||||
|
||||
private VectorAlgorithm vectorAlgorithm() {
|
||||
if (config.vectorAlgorithm == Algorithm.HSNW) {
|
||||
return VectorAlgorithm.HNSW;
|
||||
}
|
||||
return VectorAlgorithm.FLAT;
|
||||
}
|
||||
|
||||
private String jsonPath(String field) {
|
||||
return JSON_PATH_PREFIX + field;
|
||||
}
|
||||
|
||||
private static float[] toFloatArray(List<Double> embeddingDouble) {
|
||||
float[] embeddingFloat = new float[embeddingDouble.size()];
|
||||
int i = 0;
|
||||
for (Double d : embeddingDouble) {
|
||||
embeddingFloat[i++] = d.floatValue();
|
||||
}
|
||||
return embeddingFloat;
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,310 @@
|
||||
Fel 表达式引擎
|
||||
|
||||
一、名词解释
|
||||
|
||||
Fel：全称是 Fast Expression Language，一种开源表达式引擎。
|
||||
EL：表达式语言，用于求表达式的值。
|
||||
Ast：抽象语法树，一般由语法分析工具生成。1+2*3 会解析结果如下所示：
|
||||
|
||||
二、简介
|
||||
|
||||
Fel（Fast Expression Language)在源自于企业项目，设计目标是为了满足不断变化的功能需
|
||||
求和性能需求。
|
||||
|
||||
Fel 是开放的，引擎执行中的多个模块都可以扩展或替换。Fel 的执行主要是通过函数实现,
|
||||
运算符(+、-等都是 Fel 函数），所有这些函数都是可以替换的，扩展函数也非常简单。
|
||||
|
||||
Fel 有双引擎，同时支持解释执行和编译执行。可以根据性能要求选择执行方式。编译执行
|
||||
就是将表达式编译成字节码（生成 java 代码和编译模块都是可以扩展和替换的）
|
||||
|
||||
Fel 基于 Java1.5开发，适用于 Java1.5及以上版本。
|
||||
|
||||
1. 特点
|
||||
|
||||
易用性：API 使用简单，语法简洁，和 java 语法很相似。
|
||||
轻量级：整个包只有200多 KB。
|
||||
高 效：目前没有发现有开源的表达式引擎比 Fel 快。
|
||||
扩展性：采用模块化设计，可灵活控制表达式的执行过程。
|
||||
根函数：Fel 支持根函数，“$('Math')”在 Fel 中是常用的使用函数的方式。
|
||||
$函数：通过$函数，Fel 可以方便的调用工具类或对象的方法（并不需要任何附加代码），
|
||||
2. 不足
|
||||
|
||||
支持脚本：否。
|
||||
|
||||
3. 适应场景
|
||||
|
||||
Fel 适合处理海量数据，Fel 良好的扩展性可以更好的帮助用户处理数据。
|
||||
Fel 同样适用于其他需要使用表达式引擎的地方（如果工作流、公式计算、数据有效性校验
|
||||
等等）
|
||||
|
||||
三、安装
|
||||
|
||||
1. 获取 Fel
|
||||
|
||||
项目主页：http://code.google.com/p/fast-el/
|
||||
下载地址：http://code.google.com/p/fast-el/downloads/list
|
||||
|
||||
2. Jdk1.6 环境
|
||||
|
||||
使用：将 fel.jar 加入 classpath 即可。
|
||||
构建 Fel：下载 fel-all.tar.gz，解压后将 src 作为源码文件夹，并且将 lib/antlr-min.jar 加入
|
||||
classpath 即可。
|
||||
|
||||
3. Jdk1.5 环境：
|
||||
|
||||
与 jdk1.6 环境下的区别在于，需要添加 jdk 内置的 tools.jar 到 classpath。
|
||||
|
||||
四、功能
|
||||
|
||||
1. 算术表达式：
|
||||
|
||||
FelEngine fel= new FelEngineImpl();
|
||||
Object result= fel.eval("5000*12+7500");
|
||||
System.out.println(result);
|
||||
|
||||
输出结果：67500
|
||||
2. 变量
|
||||
|
||||
使用变量，其代码如下所示：
|
||||
|
||||
FelContext ctx= fel.getContext();
|
||||
ctx.set("单价", 5000);
|
||||
ctx.set("数量", 12);
|
||||
ctx.set("运费", 7500);
|
||||
Object result= fel.eval("单价*数量+运费");
|
||||
System.out.println(result);
|
||||
|
||||
输出结果：67500
|
||||
|
||||
3. 调用 JAVA 方法
|
||||
|
||||
FelEngine fel= new FelEngineImpl();
|
||||
FelContext ctx= fel.getContext();
|
||||
ctx.set("out", System.out);
|
||||
fel.eval("out.println('Hello Everybody'.substring(6))");
|
||||
|
||||
输出结果：Everybody
|
||||
|
||||
4. 自定义上下文环境
|
||||
|
||||
//负责提供气象服务的上下文环境
|
||||
FelContext ctx= new AbstractConetxt() {
|
||||
|
||||
public Object get(Object name) {
|
||||
if("天气".equals(name)){
|
||||
return "晴";
|
||||
}
|
||||
if("温度".equals(name)){
|
||||
return 25;
|
||||
}
|
||||
return null;
|
||||
|
||||
}
|
||||
};
|
||||
FelEngine fel= new FelEngineImpl(ctx);
|
||||
Object eval = fel.eval("'天气:'+天气+';温度:'+温度");
|
||||
System.out.println(eval);
|
||||
|
||||
输出结果：天气:晴;温度:25
|
||||
|
||||
5. 多层上下文环境（命名空间）
|
||||
|
||||
FelEngine fel= new FelEngineImpl();
|
||||
String costStr= "成本";
|
||||
String priceStr="价格";
|
||||
FelContext baseCtx= fel.getContext();
|
||||
//父级上下文中设置成本和价格
|
||||
baseCtx.set(costStr, 50);
|
||||
baseCtx.set(priceStr,100);
|
||||
|
||||
String exp= priceStr+"-"+costStr;
|
||||
Object baseCost= fel.eval(exp);
|
||||
System.out.println("期望利润：" + baseCost);
|
||||
|
||||
FelContext ctx= new ContextChain(baseCtx, new MapContext());
|
||||
//通货膨胀导致成本增加（子级上下文 中设置成本，会覆盖父级上下文中的成本）
|
||||
ctx.set(costStr,50+20 );
|
||||
Object allCost= fel.eval(exp, ctx);
|
||||
System.out.println("实际利润：" + allCost);
|
||||
|
||||
输出结果：
|
||||
期望利润：50
|
||||
实际利润：30
|
||||
|
||||
6. 编译执行
|
||||
|
||||
FelEngine fel= new FelEngineImpl();
|
||||
FelContext ctx= fel.getContext();
|
||||
ctx.set("单价", 5000);
|
||||
ctx.set("数量", 12);
|
||||
ctx.set("运费", 7500);
|
||||
Expression exp= fel.compile("单价*数量+运费",ctx);
|
||||
Object result= exp.eval(ctx);
|
||||
System.out.println(result);
|
||||
|
||||
执行结果：67500
|
||||
备注：适合处理海量数据，编译执行的速度基本与 Java 字节码执行速度一样快。
|
||||
|
||||
7. 自定义函数
|
||||
|
||||
//定义 hello 函数
|
||||
Function fun= new CommonFunction() {
|
||||
|
||||
public String getName() {
|
||||
return "hello";
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* 调用 hello("xxx")时执行的代码
|
||||
*/
|
||||
@Override
|
||||
public Object call(Object[] arguments) {
|
||||
|
||||
Object msg= null;
|
||||
if(arguments!= null && arguments.length>0){
|
||||
|
||||
msg= arguments[0];
|
||||
}
|
||||
return ObjectUtils.toString(msg);
|
||||
}
|
||||
|
||||
};
|
||||
FelEngine e= new FelEngineImpl();
|
||||
//添加函数到引擎中。
|
||||
e.addFun(fun);
|
||||
String exp= "hello('fel')";
|
||||
//解释执行
|
||||
Object eval = e.eval(exp);
|
||||
System.out.println("hello "+eval);
|
||||
//编译执行
|
||||
Expression compile= e.compile(exp, null);
|
||||
eval = compile.eval(null);
|
||||
System.out.println("hello "+eval);
|
||||
|
||||
执行结果：
|
||||
hello fel hello fel
|
||||
|
||||
8. 调用静态方法
|
||||
|
||||
如果你觉得上面的自定义函数也麻烦，Fel 提供的$函数可以方便的调用工具类的方法 熟悉
|
||||
jQuery 的朋友肯定知道"$"函数的威力。Fel 东施效颦，也实现了一个"$"函数,其作用是获取 class
|
||||
和创建对象。结合点操作符，可以轻易的调用工具类或对象的方法。
|
||||
|
||||
//调用 Math.min(1,2)
|
||||
FelEngine.instance.eval("$('Math').min(1,2)");
|
||||
//调用 new Foo().toString();
|
||||
FelEngine.instance.eval("$('com.greenpineyu.test.Foo.new').toString()");
|
||||
|
||||
通过"$('class').method"形式的语法，就可以调用任何等三方类包（commons lang 等）及自
|
||||
定义工具类的方法，也可以创建对象，调用对象的方法。如果有需要，还可以直接注册 Java
|
||||
Method 到函数管理器中。
|
||||
五、安全（始于 0.8 版本）
|
||||
|
||||
为了防止出现“${'System'}.exit(1)”这样的表达式导致系统崩溃。Fel 加入了安全管理器，主
|
||||
要是对方法访问进行控制。安全管理器中通过允许访问的方法列表（黑名单）和禁止访问的方
|
||||
法列表（白名单）来控制方法访问。将 "java.lang.System. *"加入到黑名单，表示 System 类的
|
||||
所以方法都不能访问。将"java.lang.Math. *"加入白名单，表示只能访问 Math 类中的方法。如
|
||||
果你不喜欢这个安全管理器，可以自己开发一个，非常简单，只需要实现一个方法就可以了。
|
||||
|
||||
六、语法
|
||||
|
||||
1. 简介
|
||||
|
||||
Fel 的语法非常简单，基本和 Java 语法没没有区别。Fel 语法是 Java 语法的一个子集，支
|
||||
持的运算符有限。如果熟悉语法的话，只需要关注一下 Fel 支持的运算即可。
|
||||
|
||||
Fel 表达式由常量、变量、函数、运算符组成。详见下文。
|
||||
|
||||
2. 常量
|
||||
|
||||
常量 说明
|
||||
Number 1,1.0 等
|
||||
Boolean true,false
|
||||
String 'abc',"abc"
|
||||
|
||||
3. 变量
|
||||
|
||||
变量命名规则与 java 变量相同。变量名由字母、数字、下划线、$组成。
|
||||
|
||||
变量 说明
|
||||
|
||||
abc 以字母开头
|
||||
|
||||
abc 以下划线开头
|
||||
|
||||
$abc 以$开头
|
||||
|
||||
变量 中文变量名也可以
|
||||
|
||||
4. 函数
|
||||
|
||||
函数名的语法规则与变量相同
|
||||
|
||||
函数 说明
|
||||
|
||||
$('Math') $是变量名，'Math'是参数
|
||||
5. 运算符
|
||||
|
||||
运算符 类型 优先级 说明
|
||||
？： 条件 三元操作符
|
||||
|| 逻辑 优 或操作
|
||||
&& 先 与操作
|
||||
==、!= 关系 级 等于、不等于
|
||||
>、<、<=、>= 从 大于、小于、大于等于、小于等于
|
||||
+、- 算术 低 加减
|
||||
*、/、% 逻辑 到 乘、除、取模
|
||||
! 高 取反操作
|
||||
|
||||
七、结构
|
||||
|
||||
1. 主要组成部分
|
||||
|
||||
引擎由四部分组成，每个组成部分都可以被替换。组成部分如下所示：
|
||||
|
||||
组件 说明
|
||||
解析器 将表达式解析成 Ast 节点
|
||||
解释执行 负责执行节点
|
||||
执行上下文 负责提供变量
|
||||
编译器 负责生成代码并编译
|
||||
|
||||
2. 主要流程
|
||||
|
||||
执行表达式的流程如下所示：
|
||||
组件 说明
|
||||
解析表达式 默认 Antlr 解析表达式，生成 Ast 节点。
|
||||
Ast 节点 由解析器生成的抽象语法树节点
|
||||
节点解释器 每一个节点都有一个解释器，负责解释节点。
|
||||
编译器 负责生成表达式对应 Java 类并编译，生成 Expression 对象
|
||||
|
||||
3. 引擎上下文
|
||||
|
||||
引擎上下文负责存取变量，它是脚本引擎与 Java 对象之间的桥梁。其结构如下所示：
|
||||
|
||||
组件 说明
|
||||
FelContext 负责存储和保存变量
|
||||
MapContext 使用 Map 来保存变量
|
||||
ContextChain 上下文链，保存两个上下文的引用。存取变量委托引用的上下文处理。
|
||||
|
||||
4. 表达式解析
|
||||
|
||||
解析表达式的过程就是将表达式解析成 Ast 节点的过程，目前 Fel 由 Antlr 负责解析表达式，
|
||||
生成 Ast 节点（FelNode),流程如下所示：
|
||||
5. 解释执行
|
||||
|
||||
FelNode(Ast 节点）中都包含一个解释器，不同的节点有不同的解释器，解释器负责求节点
|
||||
值，流程如下所示：
|
||||
|
||||
6. 编译执行
|
||||
|
||||
FelNode(Ast 节点）中都包含一个代码生成器，负责将 Fel 表达式转换成 java 表达式。编译
|
||||
模块负责将 java 表达式封装成 Java 类、编译、创建 Expression。再通过调用 Expression.eval 求
|
||||
表达式的值。其过程如下所示：
|
||||
|
||||
7. 小结
|
||||
|
||||
在上文介绍的 Fel 组件中，绝大多数组件都是可能被替换的。通过在运行时替换一些小粒
|
||||
度的组件，就可以灵活控制 Fel 的执行过程，很有实用价值。有些变化较少的组件(Antlr 解析器)
|
||||
的替换过程还不是挺方便，随着 Fel 的发展，会慢慢重构，努力使 Fel 转变成让人满意的模块
|
||||
化结构。
|
||||
|
||||
@ -153,6 +153,10 @@ spring:
|
||||
|
||||
spring:
|
||||
ai:
|
||||
vectorstore:
|
||||
redis:
|
||||
index: default-index
|
||||
prefix: "default:"
|
||||
qianfan: # 文心一言
|
||||
api-key: x0cuLZ7XsaTCU08vuJWO87Lg
|
||||
secret-key: R9mYF9dl9KASgi5RUq0FQt3wRisSnOcK
|
||||
|
||||
Loading…
Reference in New Issue
Block a user