diff --git a/agents-flex-core/src/main/java/com/agentsflex/document/BaseLoader.java b/agents-flex-core/src/main/java/com/agentsflex/document/BaseLoader.java new file mode 100644 index 0000000..901f51f --- /dev/null +++ b/agents-flex-core/src/main/java/com/agentsflex/document/BaseLoader.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2022-2023, Agents-Flex (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.agentsflex.document; + +import java.io.InputStream; + +public abstract class BaseLoader implements Loader{ + + protected Parser parser; + + public BaseLoader(Parser parser) { + this.parser = parser; + } + + @Override + public Document load() { + InputStream stream = loadInputStream(); + return parser.parse(stream); + } + + public abstract InputStream loadInputStream(); + + +} diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/Text.java b/agents-flex-core/src/main/java/com/agentsflex/document/Document.java similarity index 92% rename from agents-flex-core/src/main/java/com/agentsflex/text/Text.java rename to agents-flex-core/src/main/java/com/agentsflex/document/Document.java index a3d4ddf..85c92cd 100644 --- a/agents-flex-core/src/main/java/com/agentsflex/text/Text.java +++ b/agents-flex-core/src/main/java/com/agentsflex/document/Document.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.agentsflex.text; +package com.agentsflex.document; import com.agentsflex.util.Metadata; -public class Text extends Metadata { +public class Document extends Metadata { private String content; public String getContent() { diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/Loader.java b/agents-flex-core/src/main/java/com/agentsflex/document/Loader.java similarity index 92% rename from agents-flex-core/src/main/java/com/agentsflex/text/Loader.java rename to agents-flex-core/src/main/java/com/agentsflex/document/Loader.java index ec811ab..b887514 100644 --- a/agents-flex-core/src/main/java/com/agentsflex/text/Loader.java +++ b/agents-flex-core/src/main/java/com/agentsflex/document/Loader.java @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.agentsflex.text; +package com.agentsflex.document; public interface Loader { - Text load(); + Document load(); } diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/Parser.java b/agents-flex-core/src/main/java/com/agentsflex/document/Parser.java similarity index 90% rename from agents-flex-core/src/main/java/com/agentsflex/text/Parser.java rename to agents-flex-core/src/main/java/com/agentsflex/document/Parser.java index e1f4c1b..5da54a5 100644 --- a/agents-flex-core/src/main/java/com/agentsflex/text/Parser.java +++ b/agents-flex-core/src/main/java/com/agentsflex/document/Parser.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.agentsflex.text; +package com.agentsflex.document; import java.io.InputStream; public interface Parser { - Text parse(InputStream stream); + Document parse(InputStream stream); } diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/Splitter.java b/agents-flex-core/src/main/java/com/agentsflex/document/Splitter.java similarity index 90% rename from agents-flex-core/src/main/java/com/agentsflex/text/Splitter.java rename to agents-flex-core/src/main/java/com/agentsflex/document/Splitter.java index 224e918..bed3f71 100644 --- a/agents-flex-core/src/main/java/com/agentsflex/text/Splitter.java +++ b/agents-flex-core/src/main/java/com/agentsflex/document/Splitter.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.agentsflex.text; +package com.agentsflex.document; import java.util.List; public interface Splitter { - List split(Text text); + List split(Document text); } diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/loader/FileLoader.java b/agents-flex-core/src/main/java/com/agentsflex/document/Transformer.java similarity index 75% rename from agents-flex-core/src/main/java/com/agentsflex/text/loader/FileLoader.java rename to agents-flex-core/src/main/java/com/agentsflex/document/Transformer.java index 57bc840..509ae90 100644 --- a/agents-flex-core/src/main/java/com/agentsflex/text/loader/FileLoader.java +++ b/agents-flex-core/src/main/java/com/agentsflex/document/Transformer.java @@ -13,14 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.agentsflex.text.loader; +package com.agentsflex.document; -import com.agentsflex.text.Loader; -import com.agentsflex.text.Text; - -public class FileLoader implements Loader { - @Override - public Text load() { - return null; - } +public interface Transformer { + Document transform(Document document); } diff --git a/agents-flex-core/src/main/java/com/agentsflex/document/loader/FileLoader.java b/agents-flex-core/src/main/java/com/agentsflex/document/loader/FileLoader.java new file mode 100644 index 0000000..4c8875c --- /dev/null +++ b/agents-flex-core/src/main/java/com/agentsflex/document/loader/FileLoader.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2022-2023, Agents-Flex (fuhai999@gmail.com). + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.agentsflex.document.loader; + +import com.agentsflex.document.BaseLoader; +import com.agentsflex.document.Parser; + +import java.io.InputStream; + +public class FileLoader extends BaseLoader { + + public FileLoader(Parser parser) { + super(parser); + } + + @Override + public InputStream loadInputStream() { + return null; + } +} diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/loader/HttpLoader.java b/agents-flex-core/src/main/java/com/agentsflex/document/loader/HttpLoader.java similarity index 69% rename from agents-flex-core/src/main/java/com/agentsflex/text/loader/HttpLoader.java rename to agents-flex-core/src/main/java/com/agentsflex/document/loader/HttpLoader.java index 6a66de3..2c7473e 100644 --- a/agents-flex-core/src/main/java/com/agentsflex/text/loader/HttpLoader.java +++ b/agents-flex-core/src/main/java/com/agentsflex/document/loader/HttpLoader.java @@ -13,14 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.agentsflex.text.loader; +package com.agentsflex.document.loader; -import com.agentsflex.text.Loader; -import com.agentsflex.text.Text; +import com.agentsflex.document.BaseLoader; +import com.agentsflex.document.Parser; + +import java.io.InputStream; + +public class HttpLoader extends BaseLoader{ + + public HttpLoader(Parser parser) { + super(parser); + } -public class HttpLoader implements Loader { @Override - public Text load() { + public InputStream loadInputStream() { return null; } } diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/splitter/MarkdownSplitter.java b/agents-flex-core/src/main/java/com/agentsflex/document/splitter/MarkdownSplitter.java similarity index 81% rename from agents-flex-core/src/main/java/com/agentsflex/text/splitter/MarkdownSplitter.java rename to agents-flex-core/src/main/java/com/agentsflex/document/splitter/MarkdownSplitter.java index 5801df8..198d0f3 100644 --- a/agents-flex-core/src/main/java/com/agentsflex/text/splitter/MarkdownSplitter.java +++ b/agents-flex-core/src/main/java/com/agentsflex/document/splitter/MarkdownSplitter.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.agentsflex.text.splitter; +package com.agentsflex.document.splitter; -import com.agentsflex.text.Splitter; -import com.agentsflex.text.Text; +import com.agentsflex.document.Splitter; +import com.agentsflex.document.Document; import java.util.List; public class MarkdownSplitter implements Splitter { @Override - public List split(Text text) { + public List split(Document text) { return null; } } diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/splitter/ParagraphSplitter.java b/agents-flex-core/src/main/java/com/agentsflex/document/splitter/ParagraphSplitter.java similarity index 81% rename from agents-flex-core/src/main/java/com/agentsflex/text/splitter/ParagraphSplitter.java rename to agents-flex-core/src/main/java/com/agentsflex/document/splitter/ParagraphSplitter.java index ae9a85c..fc322fa 100644 --- a/agents-flex-core/src/main/java/com/agentsflex/text/splitter/ParagraphSplitter.java +++ b/agents-flex-core/src/main/java/com/agentsflex/document/splitter/ParagraphSplitter.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.agentsflex.text.splitter; +package com.agentsflex.document.splitter; -import com.agentsflex.text.Splitter; -import com.agentsflex.text.Text; +import com.agentsflex.document.Splitter; +import com.agentsflex.document.Document; import java.util.List; public class ParagraphSplitter implements Splitter { @Override - public List split(Text text) { + public List split(Document text) { return null; } } diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/splitter/SimpleSplitter.java b/agents-flex-core/src/main/java/com/agentsflex/document/splitter/SimpleSplitter.java similarity index 82% rename from agents-flex-core/src/main/java/com/agentsflex/text/splitter/SimpleSplitter.java rename to agents-flex-core/src/main/java/com/agentsflex/document/splitter/SimpleSplitter.java index fd40d94..0bd2b80 100644 --- a/agents-flex-core/src/main/java/com/agentsflex/text/splitter/SimpleSplitter.java +++ b/agents-flex-core/src/main/java/com/agentsflex/document/splitter/SimpleSplitter.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.agentsflex.text.splitter; +package com.agentsflex.document.splitter; -import com.agentsflex.text.Splitter; -import com.agentsflex.text.Text; +import com.agentsflex.document.Splitter; +import com.agentsflex.document.Document; import com.agentsflex.util.StringUtil; import java.util.ArrayList; @@ -32,14 +32,14 @@ public class SimpleSplitter implements Splitter { } @Override - public List split(Text text) { + public List split(Document text) { if (text == null || StringUtil.noText(text.getContent())) { return Collections.emptyList(); } String[] textArray = text.getContent().split(regex); - List texts = new ArrayList<>(textArray.length); + List texts = new ArrayList<>(textArray.length); for (String textString : textArray) { - Text newText = new Text(); + Document newText = new Document(); newText.setMetadataMap(text.getMetadataMap()); newText.setContent(textString); texts.add(newText); diff --git a/agents-flex-core/src/main/java/com/agentsflex/llm/Embeddings.java b/agents-flex-core/src/main/java/com/agentsflex/llm/Embeddings.java index 24a6faf..0d510a2 100644 --- a/agents-flex-core/src/main/java/com/agentsflex/llm/Embeddings.java +++ b/agents-flex-core/src/main/java/com/agentsflex/llm/Embeddings.java @@ -15,11 +15,11 @@ */ package com.agentsflex.llm; -import com.agentsflex.text.Text; +import com.agentsflex.document.Document; import com.agentsflex.vector.VectorData; public interface Embeddings { - VectorData embeddings(Text prompt); + VectorData embeddings(Document prompt); } diff --git a/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLLmUtil.java b/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLLmUtil.java index fb6f35b..86106cc 100644 --- a/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLLmUtil.java +++ b/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLLmUtil.java @@ -22,7 +22,7 @@ import com.agentsflex.message.HumanMessage; import com.agentsflex.message.Message; import com.agentsflex.message.MessageStatus; import com.agentsflex.prompt.Prompt; -import com.agentsflex.text.Text; +import com.agentsflex.document.Document; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONPath; @@ -47,7 +47,7 @@ public class OpenAiLLmUtil { } - public static String promptToEmbeddingsPayload(Text text) { + public static String promptToEmbeddingsPayload(Document text) { // https://platform.openai.com/docs/api-reference/making-requests String payload = "{\n" + diff --git a/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLlm.java b/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLlm.java index 85e1267..8e9de8d 100644 --- a/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLlm.java +++ b/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLlm.java @@ -24,7 +24,7 @@ import com.agentsflex.llm.BaseLlm; import com.agentsflex.llm.ChatListener; import com.agentsflex.llm.FunctionCalling; import com.agentsflex.prompt.Prompt; -import com.agentsflex.text.Text; +import com.agentsflex.document.Document; import com.agentsflex.util.OKHttpUtil; import com.agentsflex.util.StringUtil; import com.agentsflex.vector.VectorData; @@ -61,7 +61,7 @@ public class OpenAiLlm extends BaseLlm implements FunctionCalli @Override - public VectorData embeddings(Text text) { + public VectorData embeddings(Document text) { Map headers = new HashMap<>(); headers.put("Content-Type", "application/json"); headers.put("Authorization", "Bearer " + getConfig().getApiKey()); diff --git a/agents-flex-llm/agents-flex-llm-qwen/src/main/java/com/agentsflex/llm/qwen/QwenLlm.java b/agents-flex-llm/agents-flex-llm-qwen/src/main/java/com/agentsflex/llm/qwen/QwenLlm.java index 934c482..c2b0403 100644 --- a/agents-flex-llm/agents-flex-llm-qwen/src/main/java/com/agentsflex/llm/qwen/QwenLlm.java +++ b/agents-flex-llm/agents-flex-llm-qwen/src/main/java/com/agentsflex/llm/qwen/QwenLlm.java @@ -23,7 +23,7 @@ import com.agentsflex.llm.BaseLlm; import com.agentsflex.llm.ChatListener; import com.agentsflex.message.AiMessage; import com.agentsflex.prompt.Prompt; -import com.agentsflex.text.Text; +import com.agentsflex.document.Document; import com.agentsflex.vector.VectorData; import java.util.HashMap; @@ -62,7 +62,7 @@ public class QwenLlm extends BaseLlm { @Override - public VectorData embeddings(Text text) { + public VectorData embeddings(Document text) { return null; } } diff --git a/agents-flex-llm/agents-flex-llm-spark/src/main/java/com/agentsflex/llm/spark/SparkLlm.java b/agents-flex-llm/agents-flex-llm-spark/src/main/java/com/agentsflex/llm/spark/SparkLlm.java index 5c3e792..d2f6fd1 100644 --- a/agents-flex-llm/agents-flex-llm-spark/src/main/java/com/agentsflex/llm/spark/SparkLlm.java +++ b/agents-flex-llm/agents-flex-llm-spark/src/main/java/com/agentsflex/llm/spark/SparkLlm.java @@ -22,7 +22,7 @@ import com.agentsflex.llm.client.impl.WebSocketClient; import com.agentsflex.llm.BaseLlm; import com.agentsflex.llm.ChatListener; import com.agentsflex.prompt.Prompt; -import com.agentsflex.text.Text; +import com.agentsflex.document.Document; import com.agentsflex.vector.VectorData; public class SparkLlm extends BaseLlm { @@ -46,7 +46,7 @@ public class SparkLlm extends BaseLlm { @Override - public VectorData embeddings(Text text) { + public VectorData embeddings(Document text) { return null; } }