diff --git a/agents-flex-core/src/main/java/com/agentsflex/document/BaseLoader.java b/agents-flex-core/src/main/java/com/agentsflex/document/BaseLoader.java
new file mode 100644
index 0000000..901f51f
--- /dev/null
+++ b/agents-flex-core/src/main/java/com/agentsflex/document/BaseLoader.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2022-2023, Agents-Flex (fuhai999@gmail.com).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.agentsflex.document;
+
+import java.io.InputStream;
+
+public abstract class BaseLoader implements Loader{
+
+ protected Parser parser;
+
+ public BaseLoader(Parser parser) {
+ this.parser = parser;
+ }
+
+ @Override
+ public Document load() {
+ InputStream stream = loadInputStream();
+ return parser.parse(stream);
+ }
+
+ public abstract InputStream loadInputStream();
+
+
+}
diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/Text.java b/agents-flex-core/src/main/java/com/agentsflex/document/Document.java
similarity index 92%
rename from agents-flex-core/src/main/java/com/agentsflex/text/Text.java
rename to agents-flex-core/src/main/java/com/agentsflex/document/Document.java
index a3d4ddf..85c92cd 100644
--- a/agents-flex-core/src/main/java/com/agentsflex/text/Text.java
+++ b/agents-flex-core/src/main/java/com/agentsflex/document/Document.java
@@ -13,11 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.agentsflex.text;
+package com.agentsflex.document;
import com.agentsflex.util.Metadata;
-public class Text extends Metadata {
+public class Document extends Metadata {
private String content;
public String getContent() {
diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/Loader.java b/agents-flex-core/src/main/java/com/agentsflex/document/Loader.java
similarity index 92%
rename from agents-flex-core/src/main/java/com/agentsflex/text/Loader.java
rename to agents-flex-core/src/main/java/com/agentsflex/document/Loader.java
index ec811ab..b887514 100644
--- a/agents-flex-core/src/main/java/com/agentsflex/text/Loader.java
+++ b/agents-flex-core/src/main/java/com/agentsflex/document/Loader.java
@@ -13,8 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.agentsflex.text;
+package com.agentsflex.document;
public interface Loader {
- Text load();
+ Document load();
}
diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/Parser.java b/agents-flex-core/src/main/java/com/agentsflex/document/Parser.java
similarity index 90%
rename from agents-flex-core/src/main/java/com/agentsflex/text/Parser.java
rename to agents-flex-core/src/main/java/com/agentsflex/document/Parser.java
index e1f4c1b..5da54a5 100644
--- a/agents-flex-core/src/main/java/com/agentsflex/text/Parser.java
+++ b/agents-flex-core/src/main/java/com/agentsflex/document/Parser.java
@@ -13,10 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.agentsflex.text;
+package com.agentsflex.document;
import java.io.InputStream;
public interface Parser {
- Text parse(InputStream stream);
+ Document parse(InputStream stream);
}
diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/Splitter.java b/agents-flex-core/src/main/java/com/agentsflex/document/Splitter.java
similarity index 90%
rename from agents-flex-core/src/main/java/com/agentsflex/text/Splitter.java
rename to agents-flex-core/src/main/java/com/agentsflex/document/Splitter.java
index 224e918..bed3f71 100644
--- a/agents-flex-core/src/main/java/com/agentsflex/text/Splitter.java
+++ b/agents-flex-core/src/main/java/com/agentsflex/document/Splitter.java
@@ -13,10 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.agentsflex.text;
+package com.agentsflex.document;
import java.util.List;
public interface Splitter {
- List split(Text text);
+ List split(Document text);
}
diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/loader/FileLoader.java b/agents-flex-core/src/main/java/com/agentsflex/document/Transformer.java
similarity index 75%
rename from agents-flex-core/src/main/java/com/agentsflex/text/loader/FileLoader.java
rename to agents-flex-core/src/main/java/com/agentsflex/document/Transformer.java
index 57bc840..509ae90 100644
--- a/agents-flex-core/src/main/java/com/agentsflex/text/loader/FileLoader.java
+++ b/agents-flex-core/src/main/java/com/agentsflex/document/Transformer.java
@@ -13,14 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.agentsflex.text.loader;
+package com.agentsflex.document;
-import com.agentsflex.text.Loader;
-import com.agentsflex.text.Text;
-
-public class FileLoader implements Loader {
- @Override
- public Text load() {
- return null;
- }
+public interface Transformer {
+ Document transform(Document document);
}
diff --git a/agents-flex-core/src/main/java/com/agentsflex/document/loader/FileLoader.java b/agents-flex-core/src/main/java/com/agentsflex/document/loader/FileLoader.java
new file mode 100644
index 0000000..4c8875c
--- /dev/null
+++ b/agents-flex-core/src/main/java/com/agentsflex/document/loader/FileLoader.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2022-2023, Agents-Flex (fuhai999@gmail.com).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.agentsflex.document.loader;
+
+import com.agentsflex.document.BaseLoader;
+import com.agentsflex.document.Parser;
+
+import java.io.InputStream;
+
+public class FileLoader extends BaseLoader {
+
+ public FileLoader(Parser parser) {
+ super(parser);
+ }
+
+ @Override
+ public InputStream loadInputStream() {
+ return null;
+ }
+}
diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/loader/HttpLoader.java b/agents-flex-core/src/main/java/com/agentsflex/document/loader/HttpLoader.java
similarity index 69%
rename from agents-flex-core/src/main/java/com/agentsflex/text/loader/HttpLoader.java
rename to agents-flex-core/src/main/java/com/agentsflex/document/loader/HttpLoader.java
index 6a66de3..2c7473e 100644
--- a/agents-flex-core/src/main/java/com/agentsflex/text/loader/HttpLoader.java
+++ b/agents-flex-core/src/main/java/com/agentsflex/document/loader/HttpLoader.java
@@ -13,14 +13,21 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.agentsflex.text.loader;
+package com.agentsflex.document.loader;
-import com.agentsflex.text.Loader;
-import com.agentsflex.text.Text;
+import com.agentsflex.document.BaseLoader;
+import com.agentsflex.document.Parser;
+
+import java.io.InputStream;
+
+public class HttpLoader extends BaseLoader{
+
+ public HttpLoader(Parser parser) {
+ super(parser);
+ }
-public class HttpLoader implements Loader {
@Override
- public Text load() {
+ public InputStream loadInputStream() {
return null;
}
}
diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/splitter/MarkdownSplitter.java b/agents-flex-core/src/main/java/com/agentsflex/document/splitter/MarkdownSplitter.java
similarity index 81%
rename from agents-flex-core/src/main/java/com/agentsflex/text/splitter/MarkdownSplitter.java
rename to agents-flex-core/src/main/java/com/agentsflex/document/splitter/MarkdownSplitter.java
index 5801df8..198d0f3 100644
--- a/agents-flex-core/src/main/java/com/agentsflex/text/splitter/MarkdownSplitter.java
+++ b/agents-flex-core/src/main/java/com/agentsflex/document/splitter/MarkdownSplitter.java
@@ -13,16 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.agentsflex.text.splitter;
+package com.agentsflex.document.splitter;
-import com.agentsflex.text.Splitter;
-import com.agentsflex.text.Text;
+import com.agentsflex.document.Splitter;
+import com.agentsflex.document.Document;
import java.util.List;
public class MarkdownSplitter implements Splitter {
@Override
- public List split(Text text) {
+ public List split(Document text) {
return null;
}
}
diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/splitter/ParagraphSplitter.java b/agents-flex-core/src/main/java/com/agentsflex/document/splitter/ParagraphSplitter.java
similarity index 81%
rename from agents-flex-core/src/main/java/com/agentsflex/text/splitter/ParagraphSplitter.java
rename to agents-flex-core/src/main/java/com/agentsflex/document/splitter/ParagraphSplitter.java
index ae9a85c..fc322fa 100644
--- a/agents-flex-core/src/main/java/com/agentsflex/text/splitter/ParagraphSplitter.java
+++ b/agents-flex-core/src/main/java/com/agentsflex/document/splitter/ParagraphSplitter.java
@@ -13,16 +13,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.agentsflex.text.splitter;
+package com.agentsflex.document.splitter;
-import com.agentsflex.text.Splitter;
-import com.agentsflex.text.Text;
+import com.agentsflex.document.Splitter;
+import com.agentsflex.document.Document;
import java.util.List;
public class ParagraphSplitter implements Splitter {
@Override
- public List split(Text text) {
+ public List split(Document text) {
return null;
}
}
diff --git a/agents-flex-core/src/main/java/com/agentsflex/text/splitter/SimpleSplitter.java b/agents-flex-core/src/main/java/com/agentsflex/document/splitter/SimpleSplitter.java
similarity index 82%
rename from agents-flex-core/src/main/java/com/agentsflex/text/splitter/SimpleSplitter.java
rename to agents-flex-core/src/main/java/com/agentsflex/document/splitter/SimpleSplitter.java
index fd40d94..0bd2b80 100644
--- a/agents-flex-core/src/main/java/com/agentsflex/text/splitter/SimpleSplitter.java
+++ b/agents-flex-core/src/main/java/com/agentsflex/document/splitter/SimpleSplitter.java
@@ -13,10 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package com.agentsflex.text.splitter;
+package com.agentsflex.document.splitter;
-import com.agentsflex.text.Splitter;
-import com.agentsflex.text.Text;
+import com.agentsflex.document.Splitter;
+import com.agentsflex.document.Document;
import com.agentsflex.util.StringUtil;
import java.util.ArrayList;
@@ -32,14 +32,14 @@ public class SimpleSplitter implements Splitter {
}
@Override
- public List split(Text text) {
+ public List split(Document text) {
if (text == null || StringUtil.noText(text.getContent())) {
return Collections.emptyList();
}
String[] textArray = text.getContent().split(regex);
- List texts = new ArrayList<>(textArray.length);
+ List texts = new ArrayList<>(textArray.length);
for (String textString : textArray) {
- Text newText = new Text();
+ Document newText = new Document();
newText.setMetadataMap(text.getMetadataMap());
newText.setContent(textString);
texts.add(newText);
diff --git a/agents-flex-core/src/main/java/com/agentsflex/llm/Embeddings.java b/agents-flex-core/src/main/java/com/agentsflex/llm/Embeddings.java
index 24a6faf..0d510a2 100644
--- a/agents-flex-core/src/main/java/com/agentsflex/llm/Embeddings.java
+++ b/agents-flex-core/src/main/java/com/agentsflex/llm/Embeddings.java
@@ -15,11 +15,11 @@
*/
package com.agentsflex.llm;
-import com.agentsflex.text.Text;
+import com.agentsflex.document.Document;
import com.agentsflex.vector.VectorData;
public interface Embeddings {
- VectorData embeddings(Text prompt);
+ VectorData embeddings(Document prompt);
}
diff --git a/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLLmUtil.java b/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLLmUtil.java
index fb6f35b..86106cc 100644
--- a/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLLmUtil.java
+++ b/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLLmUtil.java
@@ -22,7 +22,7 @@ import com.agentsflex.message.HumanMessage;
import com.agentsflex.message.Message;
import com.agentsflex.message.MessageStatus;
import com.agentsflex.prompt.Prompt;
-import com.agentsflex.text.Text;
+import com.agentsflex.document.Document;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSONPath;
@@ -47,7 +47,7 @@ public class OpenAiLLmUtil {
}
- public static String promptToEmbeddingsPayload(Text text) {
+ public static String promptToEmbeddingsPayload(Document text) {
// https://platform.openai.com/docs/api-reference/making-requests
String payload = "{\n" +
diff --git a/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLlm.java b/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLlm.java
index 85e1267..8e9de8d 100644
--- a/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLlm.java
+++ b/agents-flex-llm/agents-flex-llm-openai/src/main/java/com/agentsflex/llm/openai/OpenAiLlm.java
@@ -24,7 +24,7 @@ import com.agentsflex.llm.BaseLlm;
import com.agentsflex.llm.ChatListener;
import com.agentsflex.llm.FunctionCalling;
import com.agentsflex.prompt.Prompt;
-import com.agentsflex.text.Text;
+import com.agentsflex.document.Document;
import com.agentsflex.util.OKHttpUtil;
import com.agentsflex.util.StringUtil;
import com.agentsflex.vector.VectorData;
@@ -61,7 +61,7 @@ public class OpenAiLlm extends BaseLlm implements FunctionCalli
@Override
- public VectorData embeddings(Text text) {
+ public VectorData embeddings(Document text) {
Map headers = new HashMap<>();
headers.put("Content-Type", "application/json");
headers.put("Authorization", "Bearer " + getConfig().getApiKey());
diff --git a/agents-flex-llm/agents-flex-llm-qwen/src/main/java/com/agentsflex/llm/qwen/QwenLlm.java b/agents-flex-llm/agents-flex-llm-qwen/src/main/java/com/agentsflex/llm/qwen/QwenLlm.java
index 934c482..c2b0403 100644
--- a/agents-flex-llm/agents-flex-llm-qwen/src/main/java/com/agentsflex/llm/qwen/QwenLlm.java
+++ b/agents-flex-llm/agents-flex-llm-qwen/src/main/java/com/agentsflex/llm/qwen/QwenLlm.java
@@ -23,7 +23,7 @@ import com.agentsflex.llm.BaseLlm;
import com.agentsflex.llm.ChatListener;
import com.agentsflex.message.AiMessage;
import com.agentsflex.prompt.Prompt;
-import com.agentsflex.text.Text;
+import com.agentsflex.document.Document;
import com.agentsflex.vector.VectorData;
import java.util.HashMap;
@@ -62,7 +62,7 @@ public class QwenLlm extends BaseLlm {
@Override
- public VectorData embeddings(Text text) {
+ public VectorData embeddings(Document text) {
return null;
}
}
diff --git a/agents-flex-llm/agents-flex-llm-spark/src/main/java/com/agentsflex/llm/spark/SparkLlm.java b/agents-flex-llm/agents-flex-llm-spark/src/main/java/com/agentsflex/llm/spark/SparkLlm.java
index 5c3e792..d2f6fd1 100644
--- a/agents-flex-llm/agents-flex-llm-spark/src/main/java/com/agentsflex/llm/spark/SparkLlm.java
+++ b/agents-flex-llm/agents-flex-llm-spark/src/main/java/com/agentsflex/llm/spark/SparkLlm.java
@@ -22,7 +22,7 @@ import com.agentsflex.llm.client.impl.WebSocketClient;
import com.agentsflex.llm.BaseLlm;
import com.agentsflex.llm.ChatListener;
import com.agentsflex.prompt.Prompt;
-import com.agentsflex.text.Text;
+import com.agentsflex.document.Document;
import com.agentsflex.vector.VectorData;
public class SparkLlm extends BaseLlm {
@@ -46,7 +46,7 @@ public class SparkLlm extends BaseLlm {
@Override
- public VectorData embeddings(Text text) {
+ public VectorData embeddings(Document text) {
return null;
}
}