当前位置：首页 > news >正文

Es bulk批量导入数据(1w+以上)

news 2025/7/28 0:57:16

最近在学习es的理论知识以及实际操作，随时更新~
概要：首先你得有1w条数据的json，然后用java读取json文件导入

一. 创建Json数据

首先我生成1.5w条数据，是为了实践分页查询，用from-size和scroll翻页去实践
生成四个字段，name、age、sex、telephone

代码如下：可直接复制粘贴用

package es;import java.io.FileWriter;
import java.io.IOException;
import java.util.Random;/*** 生成1.5W条json数据*/
public class JsonGenerator {public static void main(String[] args) {int numberOfRecords = 15000;String outputFile = "user.json";try (FileWriter writer = new FileWriter(outputFile)) {for (int i = 0; i < numberOfRecords; i++) {writer.write(generateJsonRecord());if (i < numberOfRecords - 1) {writer.write(",");} else {writer.write(" ");}}System.out.println("Generated " + numberOfRecords + " records successfully.");} catch (IOException e) {e.printStackTrace();}}private static String generateJsonRecord() {String name = generateRandomName();int age = generateRandomAge();String sex = generateRandomSex();String telephone = generateRandomTelephone();return "{\"name\":\"" + name + "\",\"age\":\"" + age + "\",\"sex\":\"" + sex + "\",\"telephone\":\"" + telephone + "\"}";}private static String generateRandomName() {String characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";StringBuilder sb = new StringBuilder();Random random = new Random();for (int i = 0; i < 5; i++) {sb.append(characters.charAt(random.nextInt(characters.length())));}return sb.toString();}private static int generateRandomAge() {Random random = new Random();return random.nextInt(65) + 18;  // Generate age between 18 and 65}private static String generateRandomSex() {Random random = new Random();return random.nextBoolean() ? "Male" : "Female";}private static String generateRandomTelephone() {StringBuilder sb = new StringBuilder();Random random = new Random();for (int i = 0; i < 10; i++) {sb.append(random.nextInt(10));  // Append random digit to the telephone number}sb.append("-");for (int i = 0; i < 3; i++) {sb.append(random.nextInt(10));  // Append random digit to the telephone number}return sb.toString();}
}

生成的文件在该位置上：
在这里插入图片描述

二.bulk 批量导入数据

public class EsConnectionExample {                                   public static void main(String[] args) throws IOException {      // 创建客户端                                                                                                            RestHighLevelClient client = new RestHighLevelClient(                                                               RestClient.builder(                                                                                         new HttpHost("ip", port, "http"))); // 修改为你的ES地址和端口                                      //bulk 导入  //这块改成你的文件的地址                                                                                         try (BufferedReader reader = new BufferedReader(new FileReader("user.json"))) {                     String line;                                                                                    // 构造 BulkRequest 对象并添加要导入的文档                                                                   BulkRequest request = new BulkRequest();                                                        while ((line = reader.readLine()) != null) {                                                    XContentBuilder builder = XContentFactory.jsonBuilder()                                     .startObject()                                                                      .field("name", line)                                                                .field("age", line)                                                                 .field("sex" , line)                                                                .field("telephone", line)                                                           .endObject();   //这块改成你的索引名字                                                                    IndexRequest indexRequest = new IndexRequest("my_index")                                    .source(builder);                                                                   request.add(indexRequest);                                                                  }                                                                                               // 发送 BulkRequest 请求                                                                            BulkResponse response = client.bulk(request, RequestOptions.DEFAULT);                           if (response.hasFailures()) {                                                                   System.out.println("Failed to import documents.");                                          } else {                                                                                        System.out.println("Documents imported successfully!");                                     }                                                                                               } catch (IOException e) {                                                                           e.printStackTrace();                                                                            } finally {                                                                                         // 关闭 ElasticSearch 客户端连接                                                                       client.close();                                                                                 } }                                                                                                  }