wx-group / juvenile-prosecution

Commit 7f4d2506c5bf15095e320e782956ad2e7faf2032

Authored by wxy
2 parents 8312c0f7 ed7d30d0

解决冲突

Inline Side-by-side
Showing 6 changed files with 709 additions and 133 deletions
juvenile-prosecution-boot/.gitignore
  View file @7f4d250
  1 +/target/
  2 +/.idea/
  3 +*.iml
  4 +rebel.xml
0 \ No newline at end of file 5 \ No newline at end of file
juvenile-prosecution-boot/jeecg-boot-module-system/pom.xml
  View file @7f4d250
@@ -45,7 +45,49 @@ @@ -45,7 +45,49 @@
45 <version>1.4.7</version> 45 <version>1.4.7</version>
46 </dependency> 46 </dependency>
47 47
48 - <!--<dependency>--> 48 + <dependency>
  49 + <groupId>org.apache.spark</groupId>
  50 + <artifactId>spark-core_2.12</artifactId>
  51 + <version>3.3.0</version>
  52 + <exclusions>
  53 + <exclusion>
  54 + <groupId>org.apache.logging.log4j</groupId>
  55 + <artifactId>log4j-slf4j-impl</artifactId>
  56 + </exclusion>
  57 + <exclusion>
  58 + <groupId>org.apache.logging.log4j</groupId>
  59 + <artifactId>log4j-api</artifactId>
  60 + </exclusion>
  61 + <!--<exclusion>-->
  62 + <!--<groupId>org.apache.logging.log4j</groupId>-->
  63 + <!--<artifactId>log4j-core</artifactId>-->
  64 + <!--</exclusion>-->
  65 + <exclusion>
  66 + <groupId>org.apache.logging.log4j</groupId>
  67 + <artifactId>log4j-1.2-api</artifactId>
  68 + </exclusion>
  69 + </exclusions>
  70 + </dependency>
  71 +
  72 + <dependency>
  73 + <groupId>org.apache.spark</groupId>
  74 + <artifactId>spark-hive_2.12</artifactId>
  75 + <version>3.3.0</version>
  76 + </dependency>
  77 +
  78 + <dependency>
  79 + <groupId>org.apache.spark</groupId>
  80 + <artifactId>spark-sql_2.12</artifactId>
  81 + <version>3.3.0</version>
  82 + </dependency>
  83 +
  84 + <dependency>
  85 + <groupId>org.codehaus.janino</groupId>
  86 + <artifactId>janino</artifactId>
  87 + <version>3.0.8</version>
  88 + </dependency>
  89 +
  90 + <!--<dependency>-->
49 <!--<groupId>org.jeecgframework.boot</groupId>--> 91 <!--<groupId>org.jeecgframework.boot</groupId>-->
50 <!--<artifactId>jeecg-boot-module-demo</artifactId>--> 92 <!--<artifactId>jeecg-boot-module-demo</artifactId>-->
51 <!--<version>${jeecgboot.version}</version>--> 93 <!--<version>${jeecgboot.version}</version>-->
juvenile-prosecution-boot/jeecg-boot-module-system/src/main/java/org/jeecg/modules/handle/SparkUtil.java 0 → 100644
  View file @7f4d250
  1 +package org.jeecg.modules.handle;
  2 +
  3 +import cn.hutool.core.map.MapUtil;
  4 +import org.apache.hadoop.conf.Configuration;
  5 +import org.apache.spark.sql.SparkSession;
  6 +
  7 +import java.net.URL;
  8 +import java.util.Iterator;
  9 +import java.util.Map;
  10 +
  11 +/**
  12 + * 类描述：Spark工具类
  13 + *
  14 + * @Author shaohu
  15 + * @Date 2022-07-18 16:55
  16 + */
  17 +public class SparkUtil {
  18 +
  19 + private static SparkSession spark;
  20 + private static SparkSession.Builder builder;
  21 + private static Configuration config;
  22 +
  23 + static {
  24 + builder = SparkSession
  25 + .builder()
  26 + .master("local[*]")
  27 + .appName("Spark Prosecution")
  28 + .config("spark.ui.enabled", false);
  29 + //.config("spark.driver.memory","2g")
  30 + }
  31 +
  32 + /**
  33 + * 配置spark环境
  34 + *
  35 + * @param
  36 + * @return
  37 + */
  38 + public static void setConfig(String key, String value) {
  39 + if (config == null) {
  40 + config = new Configuration();
  41 + }
  42 + config.set(key, value);
  43 + }
  44 +
  45 + /**
  46 + * 配置spark环境
  47 + *
  48 + * @param configMap
  49 + */
  50 + public static void setConfig(Map<String, String> configMap) {
  51 + if (config == null) {
  52 + config = new Configuration();
  53 + }
  54 + if (MapUtil.isNotEmpty(configMap)) {
  55 + configMap.entrySet().stream().forEach(entry -> {
  56 + String key = entry.getKey();
  57 + String value = entry.getValue();
  58 + config.set(key, value);
  59 + });
  60 + }
  61 + }
  62 +
  63 + /**
  64 + * 添加资源，比如hive-site.xml
  65 + *
  66 + * @param url
  67 + */
  68 + public static void addResource(URL url) {
  69 + if (config == null) {
  70 + config = new Configuration();
  71 + }
  72 + config.addResource(url);
  73 + }
  74 +
  75 + /**
  76 + * 支持hive
  77 + */
  78 + public static void enableHiveSupport() {
  79 + builder.enableHiveSupport();
  80 + }
  81 +
  82 +
  83 + /**
  84 + * 获取spark环境
  85 + *
  86 + * @return
  87 + */
  88 + public static SparkSession getSparkInstance() {
  89 + if (spark == null) {
  90 + if (config != null) {
  91 + Iterator<Map.Entry<String, String>> iterator = config.iterator();
  92 + while (iterator.hasNext()) {
  93 + Map.Entry<String, String> configEntry = iterator.next();
  94 + builder.config(configEntry.getKey(), configEntry.getValue());
  95 + }
  96 + }
  97 + spark = builder.getOrCreate();
  98 + //spark.sparkContext().setLogLevel(Level.ERROR.name());
  99 + }
  100 + return spark;
  101 + }
  102 +
  103 + /**
  104 + * 销毁spark
  105 + */
  106 + public static void close() {
  107 + spark.close();
  108 + }
  109 +
  110 +}
juvenile-prosecution-boot/jeecg-boot-module-system/src/main/java/org/jeecg/modules/handle/service/SparkEtlService.java 0 → 100644
  View file @7f4d250
  1 +package org.jeecg.modules.handle.service;
  2 +
  3 +import lombok.extern.slf4j.Slf4j;
  4 +import org.apache.spark.api.java.JavaRDD;
  5 +import org.apache.spark.api.java.JavaSparkContext;
  6 +import org.apache.spark.api.java.function.Function;
  7 +import org.apache.spark.sql.Dataset;
  8 +import org.apache.spark.sql.Row;
  9 +import org.apache.spark.sql.RowFactory;
  10 +import org.apache.spark.sql.SaveMode;
  11 +import org.apache.spark.sql.types.DataTypes;
  12 +import org.apache.spark.sql.types.StructField;
  13 +import org.apache.spark.sql.types.StructType;
  14 +import org.jeecg.common.util.DateUtils;
  15 +import org.jeecg.modules.handle.SparkUtil;
  16 +import org.jeecg.modules.system.entity.Family;
  17 +import org.jeecg.modules.system.entity.GaDemographic;
  18 +import org.jeecg.modules.system.entity.Minor;
  19 +import org.jeecg.modules.system.entity.MrSchool;
  20 +import org.springframework.beans.factory.annotation.Value;
  21 +import org.springframework.stereotype.Service;
  22 +import scala.Tuple2;
  23 +
  24 +import javax.annotation.PostConstruct;
  25 +import java.io.Serializable;
  26 +import java.sql.CallableStatement;
  27 +import java.sql.Connection;
  28 +import java.sql.DriverManager;
  29 +import java.sql.SQLException;
  30 +import java.util.ArrayList;
  31 +import java.util.Date;
  32 +import java.util.List;
  33 +import java.util.Properties;
  34 +
  35 +/**
  36 + * 类描述：
  37 + *
  38 + * @Author shaohu
  39 + * @Date 2022-07-27 14:39
  40 + */
  41 +@Slf4j
  42 +@Service
  43 +public class SparkEtlService implements Serializable {
  44 +
  45 + @Value("${spark.datasource.url}")
  46 + private String jdbcUrl;
  47 + @Value("${spark.datasource.username}")
  48 + private String username;
  49 + @Value("${spark.datasource.password}")
  50 + private String password;
  51 + @Value("${spark.datasource.driver}")
  52 + private String driver;
  53 + private int SPARK_NUM_PARTITIONS = 10;
  54 +
  55 + private Properties properties = new Properties();
  56 +
  57 + //Spark数据存入Mysql 几种模式
  58 + //默认为SaveMode.ErrorIfExists模式，该模式下，若数据库中已经存在该表，则会直接报异常，导致数据不能存入数据库；
  59 + //SaveMode.Append 若表已经存在，则追加在该表中；若该表不存在，则会先创建表，再插入数据；
  60 + //SaveMode.Overwrite 重写模式，其本质是先将已有的表及其数据全都删除，再重新创建该表，然后插入新的数据；
  61 + //SaveMode.Ignore 若表不存在，则创建表，并存入数据；若表存在的情况下，直接跳过数据的存储，不会报错
  62 + private SaveMode SPARK_MODE = SaveMode.Append;
  63 +
  64 + private String TABLE_MINOR = "mr_minor_copy1";
  65 + private String TABLE_FAMILY = "mr_family_copy1";
  66 + private String TABLE_SCHOOL = "mr_school_copy1";
  67 +
  68 + @PostConstruct
  69 + private void init() {
  70 + SparkUtil.setConfig("spark.sql.shuffle.partitions", "20");
  71 + //创建Properties对象
  72 + properties.setProperty("user", username); // 用户名
  73 + properties.setProperty("password", password); // 密码
  74 + properties.setProperty("driver", driver);
  75 + //设置分区数
  76 + properties.setProperty("numPartitions", String.valueOf(SPARK_NUM_PARTITIONS));
  77 + }
  78 +
  79 + /**
  80 + * 数据入库
  81 + *
  82 + * @param minors
  83 + * @param mrSchools
  84 + */
  85 + public void dataToDb(List<Minor> minors, List<MrSchool> mrSchools) {
  86 + long startTimeMillis1 = System.currentTimeMillis();
  87 + log.info("-----------分析数据入库 开始：" + startTimeMillis1);
  88 + //家庭成员数据
  89 + List<Family> families = new ArrayList<Family>();
  90 + for (Minor minor : minors) {
  91 + if (null != minor.getFamilies() && !minor.getFamilies().isEmpty()) {
  92 + families.addAll(minor.getFamilies());
  93 + }
  94 + }
  95 + this.saveMinor(minors);
  96 + //保存家庭成员数据
  97 + if (!families.isEmpty()) {
  98 + this.saveFamilies(families);
  99 + }
  100 + this.saveMrSchool(mrSchools);
  101 + long endTimeMillis1 = System.currentTimeMillis();
  102 + log.info("-----------分析数据入库 结束：" + endTimeMillis1);
  103 + log.info("-----------分析数据入库 耗时：" + (endTimeMillis1 - startTimeMillis1));
  104 + SparkUtil.close();
  105 + minors = null;
  106 + mrSchools = null;
  107 + }
  108 +
  109 + /**
  110 + * 根据表名清空指定表
  111 + *
  112 + * @param tableName
  113 + */
  114 + private void truncateTable(String tableName) {
  115 + Connection conn = null;
  116 + try {
  117 + Class.forName(driver);
  118 + conn = DriverManager.getConnection(jdbcUrl, properties);
  119 + CallableStatement sm = conn.prepareCall("truncate table " + tableName);
  120 + sm.execute();
  121 + sm.close();
  122 + } catch (Exception e) {
  123 + log.error(e.getMessage());
  124 + } finally {
  125 + if (null != conn) {
  126 + try {
  127 + conn.close();
  128 + } catch (SQLException e) {
  129 + log.error(e.getMessage());
  130 + }
  131 + }
  132 + }
  133 + }
  134 +
  135 + /**
  136 + * 保存未成年人和家庭成员数据
  137 + *
  138 + * @param minors
  139 + */
  140 + public void saveMinor(List<Minor> minors) {
  141 + long startTimeMillis1 = System.currentTimeMillis();
  142 + log.info("-----------未成年人入库 开始：" + startTimeMillis1);
  143 + JavaSparkContext javaSparkContext = JavaSparkContext.fromSparkContext(SparkUtil.getSparkInstance().sparkContext());
  144 + JavaRDD<Minor> minorsJavaRDD = javaSparkContext.parallelize(minors, SPARK_NUM_PARTITIONS);
  145 + String createTime = DateUtils.date2Str(new Date(), DateUtils.datetimeFormat.get());
  146 +
  147 + JavaRDD<Row> rowRDD = minorsJavaRDD.zipWithUniqueId().map(new Function<Tuple2<Minor, Long>, Row>() {
  148 + @Override
  149 + public Row call(Tuple2<Minor, Long> v1) throws Exception {
  150 + String id = v1._2.toString();
  151 + Minor minor = v1._1;
  152 + String name = minor.getName();
  153 + String sys_org_code = minor.getSysOrgCode();
  154 + String household_num = minor.getHouseholdNum();
  155 + String number = minor.getNumber();
  156 + Integer gender = minor.getGender();
  157 + String address = minor.getAddress();
  158 + String identity = minor.getIdentity();
  159 + Integer school = minor.getSchool();
  160 + String school_name = minor.getSchoolName();
  161 + String start_year = minor.getStartYear();
  162 + String guardian = minor.getGuardian();
  163 + String relation = minor.getRelation();
  164 + String reason = minor.getReason();
  165 + String special_reason = minor.getSpecialReason();
  166 + String division = minor.getDivision();
  167 + String remark = minor.getRemark();
  168 + String create_by = minor.getCreateBy();
  169 + String create_time = createTime;
  170 + String update_by = minor.getUpdateBy();
  171 + String update_time = null;
  172 + return RowFactory.create(id, name, sys_org_code, household_num, number, gender, address, identity, school, school_name,
  173 + start_year, guardian, relation, reason, special_reason, division, remark, create_by, create_time, update_by, update_time);
  174 + }
  175 + });
  176 +
  177 +// schemaFields.add(DataTypes.createStructField("id", DataTypes.LongType, false));
  178 +// schemaFields.add(DataTypes.createStructField("name", DataTypes.StringType, true));
  179 +// schemaFields.add(DataTypes.createStructField("sysOrgCode", DataTypes.StringType, true));
  180 +// schemaFields.add(DataTypes.createStructField("householdNum", DataTypes.StringType, true));
  181 +// schemaFields.add(DataTypes.createStructField("number", DataTypes.StringType, true));
  182 +// schemaFields.add(DataTypes.createStructField("gender", DataTypes.IntegerType, true));
  183 +// schemaFields.add(DataTypes.createStructField("address", DataTypes.StringType, true));
  184 +// schemaFields.add(DataTypes.createStructField("identity", DataTypes.StringType, true));
  185 +// schemaFields.add(DataTypes.createStructField("school", DataTypes.IntegerType, true));
  186 +// schemaFields.add(DataTypes.createStructField("schoolName", DataTypes.StringType, true));
  187 +// schemaFields.add(DataTypes.createStructField("startYear", DataTypes.StringType, true));
  188 +// schemaFields.add(DataTypes.createStructField("guardian", DataTypes.StringType, true));
  189 +// schemaFields.add(DataTypes.createStructField("relation", DataTypes.StringType, true));
  190 +// schemaFields.add(DataTypes.createStructField("reason", DataTypes.StringType, true));
  191 +// schemaFields.add(DataTypes.createStructField("specialReason", DataTypes.StringType, true));
  192 +// schemaFields.add(DataTypes.createStructField("division", DataTypes.StringType, true));
  193 +// schemaFields.add(DataTypes.createStructField("remark", DataTypes.StringType, true));
  194 +// schemaFields.add(DataTypes.createStructField("createBy", DataTypes.StringType, true));
  195 +// schemaFields.add(DataTypes.createStructField("createTime", DataTypes.StringType, true));
  196 +// schemaFields.add(DataTypes.createStructField("updateBy", DataTypes.StringType, true));
  197 +// schemaFields.add(DataTypes.createStructField("updateTime", DataTypes.StringType, true));
  198 +
  199 + List<StructField> schemaFields = new ArrayList<StructField>();
  200 + schemaFields.add(DataTypes.createStructField("id", DataTypes.StringType, false));
  201 + schemaFields.add(DataTypes.createStructField("name", DataTypes.StringType, true));
  202 + schemaFields.add(DataTypes.createStructField("sys_org_code", DataTypes.StringType, true));
  203 + schemaFields.add(DataTypes.createStructField("household_num", DataTypes.StringType, true));
  204 + schemaFields.add(DataTypes.createStructField("number", DataTypes.StringType, true));
  205 + schemaFields.add(DataTypes.createStructField("gender", DataTypes.IntegerType, true));
  206 + schemaFields.add(DataTypes.createStructField("address", DataTypes.StringType, true));
  207 + schemaFields.add(DataTypes.createStructField("identity", DataTypes.StringType, true));
  208 + schemaFields.add(DataTypes.createStructField("school", DataTypes.IntegerType, true));
  209 + schemaFields.add(DataTypes.createStructField("school_name", DataTypes.StringType, true));
  210 + schemaFields.add(DataTypes.createStructField("start_year", DataTypes.StringType, true));
  211 + schemaFields.add(DataTypes.createStructField("guardian", DataTypes.StringType, true));
  212 + schemaFields.add(DataTypes.createStructField("relation", DataTypes.StringType, true));
  213 + schemaFields.add(DataTypes.createStructField("reason", DataTypes.StringType, true));
  214 + schemaFields.add(DataTypes.createStructField("special_reason", DataTypes.StringType, true));
  215 + schemaFields.add(DataTypes.createStructField("division", DataTypes.StringType, true));
  216 + schemaFields.add(DataTypes.createStructField("remark", DataTypes.StringType, true));
  217 + schemaFields.add(DataTypes.createStructField("create_by", DataTypes.StringType, true));
  218 + schemaFields.add(DataTypes.createStructField("create_time", DataTypes.StringType, true));
  219 + schemaFields.add(DataTypes.createStructField("update_by", DataTypes.StringType, true));
  220 + schemaFields.add(DataTypes.createStructField("update_time", DataTypes.StringType, true));
  221 + StructType schema = DataTypes.createStructType(schemaFields);
  222 +
  223 + Dataset<Row> minorsDs = SparkUtil.getSparkInstance().createDataFrame(rowRDD.rdd(), schema);
  224 + String table = TABLE_MINOR;
  225 + this.truncateTable(table);
  226 + minorsDs.write().mode(SPARK_MODE).jdbc(jdbcUrl, table, properties);
  227 + long endTimeMillis1 = System.currentTimeMillis();
  228 + log.info("-----------未成年人入库 结束：" + endTimeMillis1);
  229 + log.info("-----------未成年人入库 耗时：" + (endTimeMillis1 - startTimeMillis1));
  230 + }
  231 +
  232 + /**
  233 + * 保存家庭成员数据
  234 + *
  235 + * @param families
  236 + */
  237 + public void saveFamilies(List<Family> families) {
  238 + long startTimeMillis1 = System.currentTimeMillis();
  239 + log.info("-----------家庭成员数据入库 开始：" + startTimeMillis1);
  240 + JavaSparkContext javaSparkContext = JavaSparkContext.fromSparkContext(SparkUtil.getSparkInstance().sparkContext());
  241 + JavaRDD<Family> familiesJavaRDD = javaSparkContext.parallelize(families, SPARK_NUM_PARTITIONS);
  242 + String createTime = DateUtils.date2Str(new Date(), DateUtils.datetimeFormat.get());
  243 + JavaRDD<Row> rowRDD = familiesJavaRDD.zipWithUniqueId().map(new Function<Tuple2<Family, Long>, Row>() {
  244 + @Override
  245 + public Row call(Tuple2<Family, Long> v1) throws Exception {
  246 + String id = v1._2.toString();
  247 + Family family = v1._1;
  248 + String name = family.getName();
  249 + String sys_org_code = family.getSysOrgCode();
  250 + String household_num = family.getHouseholdNum();
  251 + Integer gender = family.getGender();
  252 + String identity = family.getIdentity();
  253 + String number = family.getNumber();
  254 + String relation = family.getRelation();
  255 + String division = family.getDivision();
  256 + String address = family.getAddress();
  257 + Integer crime = family.getCrime();
  258 + String reason = family.getReason();
  259 + String other = family.getOther();
  260 + String create_by = family.getCreateBy();
  261 + String create_time = createTime;
  262 + String update_by = family.getUpdateBy();
  263 + String update_time = null;
  264 + return RowFactory.create(id, name, sys_org_code, household_num, gender, identity, number, relation, division,
  265 + address, crime, reason, other, create_by, create_time, update_by, update_time);
  266 + }
  267 + });
  268 +
  269 + List<StructField> schemaFields = new ArrayList<StructField>();
  270 +// schemaFields.add(DataTypes.createStructField("id", DataTypes.LongType, false));
  271 +// schemaFields.add(DataTypes.createStructField("name", DataTypes.StringType, true));
  272 +// schemaFields.add(DataTypes.createStructField("sysOrgCode", DataTypes.StringType, true));
  273 +// schemaFields.add(DataTypes.createStructField("householdNum", DataTypes.StringType, true));
  274 +// schemaFields.add(DataTypes.createStructField("gender", DataTypes.IntegerType, true));
  275 +// schemaFields.add(DataTypes.createStructField("identity", DataTypes.StringType, true));
  276 +// schemaFields.add(DataTypes.createStructField("number", DataTypes.StringType, true));
  277 +// schemaFields.add(DataTypes.createStructField("relation", DataTypes.StringType, true));
  278 +// schemaFields.add(DataTypes.createStructField("division", DataTypes.StringType, true));
  279 +// schemaFields.add(DataTypes.createStructField("address", DataTypes.StringType, true));
  280 +// schemaFields.add(DataTypes.createStructField("crime", DataTypes.IntegerType, true));
  281 +// schemaFields.add(DataTypes.createStructField("reason", DataTypes.StringType, true));
  282 +// schemaFields.add(DataTypes.createStructField("other", DataTypes.StringType, true));
  283 +// schemaFields.add(DataTypes.createStructField("createTime", DataTypes.StringType, true));
  284 + schemaFields.add(DataTypes.createStructField("id", DataTypes.StringType, false));
  285 + schemaFields.add(DataTypes.createStructField("name", DataTypes.StringType, true));
  286 + schemaFields.add(DataTypes.createStructField("sys_org_code", DataTypes.StringType, true));
  287 + schemaFields.add(DataTypes.createStructField("household_num", DataTypes.StringType, true));
  288 + schemaFields.add(DataTypes.createStructField("gender", DataTypes.IntegerType, true));
  289 + schemaFields.add(DataTypes.createStructField("identity", DataTypes.StringType, true));
  290 + schemaFields.add(DataTypes.createStructField("number", DataTypes.StringType, true));
  291 + schemaFields.add(DataTypes.createStructField("relation", DataTypes.StringType, true));
  292 + schemaFields.add(DataTypes.createStructField("division", DataTypes.StringType, true));
  293 + schemaFields.add(DataTypes.createStructField("address", DataTypes.StringType, true));
  294 + schemaFields.add(DataTypes.createStructField("crime", DataTypes.IntegerType, true));
  295 + schemaFields.add(DataTypes.createStructField("reason", DataTypes.StringType, true));
  296 + schemaFields.add(DataTypes.createStructField("other", DataTypes.StringType, true));
  297 + schemaFields.add(DataTypes.createStructField("create_by", DataTypes.StringType, true));
  298 + schemaFields.add(DataTypes.createStructField("create_time", DataTypes.StringType, true));
  299 + schemaFields.add(DataTypes.createStructField("update_by", DataTypes.StringType, true));
  300 + schemaFields.add(DataTypes.createStructField("update_time", DataTypes.StringType, true));
  301 +
  302 + StructType schema = DataTypes.createStructType(schemaFields);
  303 +
  304 + Dataset<Row> familiesDs = SparkUtil.getSparkInstance().createDataFrame(rowRDD.rdd(), schema);
  305 + String table = TABLE_FAMILY;
  306 + this.truncateTable(table);
  307 + familiesDs.write().mode(SPARK_MODE).jdbc(jdbcUrl, table, properties);
  308 + long endTimeMillis1 = System.currentTimeMillis();
  309 + log.info("-----------家庭成员数据入库 结束：" + endTimeMillis1);
  310 + log.info("-----------家庭成员数据入库 耗时：" + (endTimeMillis1 - startTimeMillis1));
  311 +
  312 +
  313 + }
  314 +
  315 + /**
  316 + * 批量保存学校数据
  317 + *
  318 + * @param mrSchools
  319 + */
  320 + public void saveMrSchool(List<MrSchool> mrSchools) {
  321 + long startTimeMillis1 = System.currentTimeMillis();
  322 + log.info("-----------学校数据入库 开始：" + startTimeMillis1);
  323 + log.info("saveMrSchool 开始：" + startTimeMillis1);
  324 + JavaSparkContext javaSparkContext = JavaSparkContext.fromSparkContext(SparkUtil.getSparkInstance().sparkContext());
  325 + JavaRDD<MrSchool> mrSchoolsJavaRDD = javaSparkContext.parallelize(mrSchools, SPARK_NUM_PARTITIONS);
  326 + String createTime = DateUtils.date2Str(new Date(), DateUtils.datetimeFormat.get());
  327 + JavaRDD<Row> rowRDD = mrSchoolsJavaRDD.zipWithUniqueId().map(new Function<Tuple2<MrSchool, Long>, Row>() {
  328 + @Override
  329 + public Row call(Tuple2<MrSchool, Long> v1) throws Exception {
  330 + String id = v1._2.toString();
  331 + MrSchool mrSchool = v1._1;
  332 + String name = mrSchool.getName();
  333 + String sys_org_code = mrSchool.getSysOrgCode();
  334 + String identity = mrSchool.getIdentity();
  335 + String school = mrSchool.getSchool();
  336 + String admission_date = mrSchool.getAdmissionDate();
  337 + String class_name = mrSchool.getClassName();
  338 + String address = mrSchool.getAddress();
  339 + String member_one = mrSchool.getMemberOne();
  340 + String connect_one = mrSchool.getConnectOne();
  341 + String phone_one = mrSchool.getPhoneOne();
  342 + String member_two = mrSchool.getMemberTwo();
  343 + String connect_two = mrSchool.getConnectTwo();
  344 + String phone_two = mrSchool.getPhoneTwo();
  345 + String phone = mrSchool.getPhone();
  346 + String create_by = mrSchool.getCreateBy();
  347 + String create_time = createTime;
  348 + String update_by = mrSchool.getUpdateBy();
  349 + String update_time = null;
  350 + return RowFactory.create(id, name, sys_org_code, identity, school, admission_date, class_name,
  351 + address, member_one, connect_one, phone_one, member_two, connect_two, phone_two, phone,
  352 + create_by, create_time, update_by, update_time);
  353 + }
  354 + });
  355 +
  356 + List<StructField> schemaFields = new ArrayList<StructField>();
  357 +// schemaFields.add(DataTypes.createStructField("id", DataTypes.LongType, false));
  358 +// schemaFields.add(DataTypes.createStructField("name", DataTypes.StringType, true));
  359 +// schemaFields.add(DataTypes.createStructField("sysOrgCode", DataTypes.StringType, true));
  360 +// schemaFields.add(DataTypes.createStructField("identity", DataTypes.StringType, true));
  361 +// schemaFields.add(DataTypes.createStructField("school", DataTypes.StringType, true));
  362 +// schemaFields.add(DataTypes.createStructField("admissionDate", DataTypes.StringType, true));
  363 +// schemaFields.add(DataTypes.createStructField("className", DataTypes.StringType, true));
  364 +// schemaFields.add(DataTypes.createStructField("address", DataTypes.StringType, true));
  365 +// schemaFields.add(DataTypes.createStructField("memberOne", DataTypes.StringType, true));
  366 +// schemaFields.add(DataTypes.createStructField("connectOne", DataTypes.StringType, true));
  367 +// schemaFields.add(DataTypes.createStructField("phoneOne", DataTypes.StringType, true));
  368 +// schemaFields.add(DataTypes.createStructField("memberTwo", DataTypes.StringType, true));
  369 +// schemaFields.add(DataTypes.createStructField("connectTwo", DataTypes.StringType, true));
  370 +// schemaFields.add(DataTypes.createStructField("phoneTwo", DataTypes.StringType, true));
  371 +// schemaFields.add(DataTypes.createStructField("phone", DataTypes.StringType, true));
  372 +// schemaFields.add(DataTypes.createStructField("createTime", DataTypes.StringType, true));
  373 + schemaFields.add(DataTypes.createStructField("id", DataTypes.StringType, false));
  374 + schemaFields.add(DataTypes.createStructField("name", DataTypes.StringType, true));
  375 + schemaFields.add(DataTypes.createStructField("sys_org_code", DataTypes.StringType, true));
  376 + schemaFields.add(DataTypes.createStructField("identity", DataTypes.StringType, true));
  377 + schemaFields.add(DataTypes.createStructField("school", DataTypes.StringType, true));
  378 + schemaFields.add(DataTypes.createStructField("admission_date", DataTypes.StringType, true));
  379 + schemaFields.add(DataTypes.createStructField("class_name", DataTypes.StringType, true));
  380 + schemaFields.add(DataTypes.createStructField("address", DataTypes.StringType, true));
  381 + schemaFields.add(DataTypes.createStructField("member_one", DataTypes.StringType, true));
  382 + schemaFields.add(DataTypes.createStructField("connect_one", DataTypes.StringType, true));
  383 + schemaFields.add(DataTypes.createStructField("phone_one", DataTypes.StringType, true));
  384 + schemaFields.add(DataTypes.createStructField("member_two", DataTypes.StringType, true));
  385 + schemaFields.add(DataTypes.createStructField("connect_two", DataTypes.StringType, true));
  386 + schemaFields.add(DataTypes.createStructField("phone_two", DataTypes.StringType, true));
  387 + schemaFields.add(DataTypes.createStructField("phone", DataTypes.StringType, true));
  388 + schemaFields.add(DataTypes.createStructField("create_by", DataTypes.StringType, true));
  389 + schemaFields.add(DataTypes.createStructField("create_time", DataTypes.StringType, true));
  390 + schemaFields.add(DataTypes.createStructField("update_by", DataTypes.StringType, true));
  391 + schemaFields.add(DataTypes.createStructField("update_time", DataTypes.StringType, true));
  392 + StructType schema = DataTypes.createStructType(schemaFields);
  393 +
  394 + Dataset<Row> schoolDs = SparkUtil.getSparkInstance().createDataFrame(rowRDD.rdd(), schema);
  395 + String table = TABLE_SCHOOL;
  396 + this.truncateTable(table);
  397 + schoolDs.write().mode(SPARK_MODE).jdbc(jdbcUrl, table, properties);
  398 + long endTimeMillis1 = System.currentTimeMillis();
  399 + log.info("-----------学校数据入库 结束：" + endTimeMillis1);
  400 + log.info("-----------学校数据入库 耗时：" + (endTimeMillis1 - startTimeMillis1));
  401 + }
  402 +}
juvenile-prosecution-boot/jeecg-boot-module-system/src/main/java/org/jeecg/modules/system/controller/AnalyzeController.java
  View file @7f4d250
@@ -10,12 +10,14 @@ import org.jeecg.common.aspect.annotation.AutoLog; @@ -10,12 +10,14 @@ import org.jeecg.common.aspect.annotation.AutoLog;
10 import org.jeecg.common.system.base.controller.JeecgController; 10 import org.jeecg.common.system.base.controller.JeecgController;
11 import org.jeecg.common.system.vo.LoginUser; 11 import org.jeecg.common.system.vo.LoginUser;
12 import org.jeecg.common.util.oConvertUtils; 12 import org.jeecg.common.util.oConvertUtils;
  13 +import org.jeecg.modules.handle.service.SparkEtlService;
13 import org.jeecg.modules.system.entity.*; 14 import org.jeecg.modules.system.entity.*;
14 import org.jeecg.modules.system.service.*; 15 import org.jeecg.modules.system.service.*;
15 import org.jeecg.modules.system.util.IDNumberUtil; 16 import org.jeecg.modules.system.util.IDNumberUtil;
16 import org.jeecg.modules.system.util.getRandomId; 17 import org.jeecg.modules.system.util.getRandomId;
17 import org.jeecg.modules.system.vo.GaPunishVo; 18 import org.jeecg.modules.system.vo.GaPunishVo;
18 import org.jeecg.modules.system.vo.SpecialStudentVo; 19 import org.jeecg.modules.system.vo.SpecialStudentVo;
  20 +import org.springframework.beans.factory.annotation.Value;
19 import org.springframework.web.bind.annotation.*; 21 import org.springframework.web.bind.annotation.*;
20 22
21 import javax.annotation.Resource; 23 import javax.annotation.Resource;
@@ -23,7 +25,6 @@ import java.util.*; @@ -23,7 +25,6 @@ import java.util.*;
23 import java.util.concurrent.*; 25 import java.util.concurrent.*;
24 26
25 27
26 -  
27 /** 28 /**
28 * @Description: 数据分析 29 * @Description: 数据分析
29 * @Author: jeecg-boot 30 * @Author: jeecg-boot
@@ -70,51 +71,60 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -70,51 +71,60 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
70 @Resource 71 @Resource
71 private ISysDictService dictService; 72 private ISysDictService dictService;
72 73
  74 +
73 // private List<Family> resultFamily = new ArrayList<>();//分析后家庭的数据 75 // private List<Family> resultFamily = new ArrayList<>();//分析后家庭的数据
74 private List<Minor> resultMinor = new ArrayList<>();//分析后未成年人的信息 76 private List<Minor> resultMinor = new ArrayList<>();//分析后未成年人的信息
75 private List<MrSchool> resultSchools = new ArrayList<>();//分析后学籍的信息 77 private List<MrSchool> resultSchools = new ArrayList<>();//分析后学籍的信息
76 private Set<String> testIdCards = new HashSet<>();//存储户号 78 private Set<String> testIdCards = new HashSet<>();//存储户号
77 private static final int corePoolSize = Runtime.getRuntime().availableProcessors(); 79 private static final int corePoolSize = Runtime.getRuntime().availableProcessors();
78 private static final ThreadPoolExecutor executor = new ThreadPoolExecutor(corePoolSize, corePoolSize + 1, 10L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(1000)); 80 private static final ThreadPoolExecutor executor = new ThreadPoolExecutor(corePoolSize, corePoolSize + 1, 10L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(1000));
79 - private Integer random_number=000000; 81 + @Value("${spark.enabled}")
  82 + private Boolean sparkEnabled;
  83 + @Resource
  84 + private SparkEtlService etlService;
  85 + private Integer random_number = 000000;
  86 +
80 87
81 @AutoLog(value = "来源数据管理-分页列表查询") 88 @AutoLog(value = "来源数据管理-分页列表查询")
82 @ApiOperation(value = "来源数据管理-分页列表查询", notes = "来源数据管理-分页列表查询") 89 @ApiOperation(value = "来源数据管理-分页列表查询", notes = "来源数据管理-分页列表查询")
83 @PutMapping(value = "/doAnalyzeData") 90 @PutMapping(value = "/doAnalyzeData")
84 public void doAnalyzeData() { 91 public void doAnalyzeData() {
85 try { 92 try {
86 - System.out.println("开始数据分析"); 93 + log.info("开始数据分析");
87 long startTime = System.currentTimeMillis(); 94 long startTime = System.currentTimeMillis();
88 List<SysDepart> departs = sysDepartService.querySysDeparts(); 95 List<SysDepart> departs = sysDepartService.querySysDeparts();
89 getData(departs); 96 getData(departs);
90 - System.out.println("开始数据分析" + (System.currentTimeMillis() - startTime) / 1000 + "秒");  
91 -// System.out.println("家庭成员数据批量新增总条数" + resultFamily.size());  
92 - System.out.println("未成年人数据批量新增总条数" + resultMinor.size());  
93 - System.out.println("学籍信息数据批量新增总条数" + resultSchools.size());  
94 - long startTime2 = System.currentTimeMillis();  
95 - minorService.insertBatch(resultMinor);  
96 - System.out.println("未成年人数据批量新增所用时间" + (System.currentTimeMillis() - startTime2) / 1000 + "秒");  
97 - long startTime1 = System.currentTimeMillis(); 97 +
  98 + if (null != sparkEnabled && sparkEnabled) {
  99 + etlService.dataToDb(resultMinor, resultSchools);
  100 + } else {
  101 + log.info("开始数据分析" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
  102 +// log.info("家庭成员数据批量新增总条数" + resultFamily.size());
  103 + log.info("未成年人数据批量新增总条数" + resultMinor.size());
  104 + log.info("学籍信息数据批量新增总条数" + resultSchools.size());
  105 + long startTime2 = System.currentTimeMillis();
  106 + minorService.insertBatch(resultMinor);
  107 + log.info("未成年人数据批量新增所用时间" + (System.currentTimeMillis() - startTime2) / 1000 + "秒");
  108 + long startTime1 = System.currentTimeMillis();
98 // familyService.insertBatch(resultFamily); 109 // familyService.insertBatch(resultFamily);
99 - System.out.println("家庭成员数据批量新增所用时间" + (System.currentTimeMillis() - startTime1) / 1000 + "秒");  
100 - long startTime3 = System.currentTimeMillis();  
101 - schoolService.insertBatch(resultSchools);  
102 - System.out.println("学籍信息入库时间" + (System.currentTimeMillis() - startTime3) / 1000 + "秒"); 110 + log.info("家庭成员数据批量新增所用时间" + (System.currentTimeMillis() - startTime1) / 1000 + "秒");
  111 + long startTime3 = System.currentTimeMillis();
  112 + schoolService.insertBatch(resultSchools);
  113 + log.info("学籍信息入库时间" + (System.currentTimeMillis() - startTime3) / 1000 + "秒");
  114 + }
103 long startTime4 = System.currentTimeMillis(); 115 long startTime4 = System.currentTimeMillis();
104 getSourceData(); 116 getSourceData();
105 - System.out.println("来源数据入库时间" + (System.currentTimeMillis() - startTime4) / 1000 + "秒");  
106 - System.out.println("*********************************总用时" + (System.currentTimeMillis() - startTime) / 1000 + "秒");  
107 -// return Result.OK("操作成功"); 117 + log.info("来源数据入库时间" + (System.currentTimeMillis() - startTime4) / 1000 + "秒");
  118 + log.info("*********************************总用时" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
108 } catch (Exception e) { 119 } catch (Exception e) {
109 e.printStackTrace(); 120 e.printStackTrace();
110 log.info(String.valueOf(e)); 121 log.info(String.valueOf(e));
111 -// return Result.OK("操作失败");  
112 } 122 }
113 } 123 }
114 124
115 public void getData(List<SysDepart> departs) { 125 public void getData(List<SysDepart> departs) {
116 try { 126 try {
117 - System.out.println("开始数据分析"); 127 + log.info("开始数据分析");
118 long startTime = System.currentTimeMillis(); 128 long startTime = System.currentTimeMillis();
119 ExecutorService pool = Executors.newFixedThreadPool(50); 129 ExecutorService pool = Executors.newFixedThreadPool(50);
120 CompletableFuture<Map<String, GaPunishVo>> f1 = CompletableFuture.supplyAsync(() -> gaPunishService.queryAll(), executor); 130 CompletableFuture<Map<String, GaPunishVo>> f1 = CompletableFuture.supplyAsync(() -> gaPunishService.queryAll(), executor);
@@ -163,19 +173,19 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -163,19 +173,19 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
163 //关闭线程池 173 //关闭线程池
164 pool.shutdown(); 174 pool.shutdown();
165 Map<String, String> gaDemographics = gaDemographicService.getIdentityS(); 175 Map<String, String> gaDemographics = gaDemographicService.getIdentityS();
166 - System.out.println("线程池查询用时" + (System.currentTimeMillis() - startTime) / 1000 + "秒"); 176 + log.info("线程池查询用时" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
167 long startTime1 = System.currentTimeMillis(); 177 long startTime1 = System.currentTimeMillis();
168 //户籍信息 178 //户籍信息
169 - Map<String,Family> familyMap = gaHouseholdService.getGaHouseholds(gaDemographics); 179 + Map<String, Family> familyMap = gaHouseholdService.getGaHouseholds(gaDemographics);
170 // Map<String,Family> familyMap = queryByThread(); 180 // Map<String,Family> familyMap = queryByThread();
171 - System.out.println("户籍信息数据查询时间" + (System.currentTimeMillis() - startTime1) / 1000 + "秒");  
172 - System.out.println("判断初高中学生信息和人口基本信息的交集和差集开始时间" + (System.currentTimeMillis() - startTime) / 1000 + "秒"); 181 + log.info("户籍信息数据查询时间" + (System.currentTimeMillis() - startTime1) / 1000 + "秒");
  182 + log.info("判断初高中学生信息和人口基本信息的交集和差集开始时间" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
173 judgeHighSchoolsAndDemographics(highSchools, gaDemographics, departs, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, familyMap, gaPunishVos, mzOrphans); 183 judgeHighSchoolsAndDemographics(highSchools, gaDemographics, departs, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, familyMap, gaPunishVos, mzOrphans);
174 - System.out.println("判断初高中学生信息和人口基本信息的交集和差集结束时间" + (System.currentTimeMillis() - startTime) / 1000 + "秒");  
175 - System.out.println("判断幼小学生信息和人口基本信息的交集和差集开始时间" + (System.currentTimeMillis() - startTime) / 1000 + "秒"); 184 + log.info("判断初高中学生信息和人口基本信息的交集和差集结束时间" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
  185 + log.info("判断幼小学生信息和人口基本信息的交集和差集开始时间" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
176 judgePrimarySchoolsAndDemographics(primarySchools, gaDemographics, departs, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, familyMap, gaPunishVos, mzOrphans); 186 judgePrimarySchoolsAndDemographics(primarySchools, gaDemographics, departs, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, familyMap, gaPunishVos, mzOrphans);
177 - System.out.println("判断幼小学生信息和人口基本信息的交集和差集结束时间" + (System.currentTimeMillis() - startTime) / 1000 + "秒");  
178 - System.out.println("所有时间" + (System.currentTimeMillis() - startTime) / 1000 + "秒"); 187 + log.info("判断幼小学生信息和人口基本信息的交集和差集结束时间" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
  188 + log.info("所有时间" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
179 } catch (Exception e) { 189 } catch (Exception e) {
180 e.printStackTrace(); 190 e.printStackTrace();
181 } 191 }
@@ -197,7 +207,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -197,7 +207,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
197 // // 循环创建线程 207 // // 循环创建线程
198 // //此处调用具体的查询方法 208 // //此处调用具体的查询方法
199 // List<Callable<Map<String,Family>>> taskList = new ArrayList<Callable<Map<String,Family>>>();//创建任务 209 // List<Callable<Map<String,Family>>> taskList = new ArrayList<Callable<Map<String,Family>>>();//创建任务
200 -// System.out.println("开始查询线程总数" + runSize); 210 +// log.info("开始查询线程总数" + runSize);
201 // for (int i = 0; i < runSize; i++) { 211 // for (int i = 0; i < runSize; i++) {
202 // int index = i * count; 212 // int index = i * count;
203 // int num = count; 213 // int num = count;
@@ -205,7 +215,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -205,7 +215,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
205 // taskList.add(task); 215 // taskList.add(task);
206 // } 216 // }
207 // List<Future<Map<String,Family>>> futureList = executor.invokeAll(taskList); 217 // List<Future<Map<String,Family>>> futureList = executor.invokeAll(taskList);
208 -// System.out.println(futureList.size()); 218 +// log.info(futureList.size());
209 // if (futureList != null && futureList.size() > 0){ 219 // if (futureList != null && futureList.size() > 0){
210 // for (Future<Map<String,Family>> future:futureList) { 220 // for (Future<Map<String,Family>> future:futureList) {
211 // if(future.get() != null) { 221 // if(future.get() != null) {
@@ -216,8 +226,8 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -216,8 +226,8 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
216 // executor.shutdown();//关闭线程 226 // executor.shutdown();//关闭线程
217 // long endTime = System.currentTimeMillis(); 227 // long endTime = System.currentTimeMillis();
218 // long s = ((endTime - startTime) / 1000); 228 // long s = ((endTime - startTime) / 1000);
219 -// System.out.println(runSize + "个线程查询花了" + s + "秒" );  
220 -// System.out.println("总条数"+familyMap.size() ); 229 +// log.info(runSize + "个线程查询花了" + s + "秒" );
  230 +// log.info("总条数"+familyMap.size() );
221 // } catch (Exception e) { 231 // } catch (Exception e) {
222 // e.printStackTrace(); 232 // e.printStackTrace();
223 // log.info(String.valueOf(e)); 233 // log.info(String.valueOf(e));
@@ -232,7 +242,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -232,7 +242,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
232 long startTime = System.currentTimeMillis(); 242 long startTime = System.currentTimeMillis();
233 List<SourceData> sourceDataList = new ArrayList<>(); 243 List<SourceData> sourceDataList = new ArrayList<>();
234 try { 244 try {
235 - System.out.println("来源数据查询"); 245 + log.info("来源数据查询");
236 246
237 //公安 247 //公安
238 Integer gaTotal = gaDemographicService.count() + gaHouseholdService.count() + gaPunishService.count() + gaJuvenilesStayService.count(); 248 Integer gaTotal = gaDemographicService.count() + gaHouseholdService.count() + gaPunishService.count() + gaJuvenilesStayService.count();
@@ -290,7 +300,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -290,7 +300,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
290 e.printStackTrace(); 300 e.printStackTrace();
291 log.info(String.valueOf(e)); 301 log.info(String.valueOf(e));
292 } 302 }
293 - System.out.println("来源数据查询" + (System.currentTimeMillis() - startTime) / 1000 + "秒"); 303 + log.info("来源数据查询" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
294 // return sourceDataList; 304 // return sourceDataList;
295 } 305 }
296 306
@@ -301,9 +311,9 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -301,9 +311,9 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
301 public void judgeHighSchoolsAndDemographics(List<MrSchool> highSchools, Map<String, String> gaDemographics, List<SysDepart> departs, 311 public void judgeHighSchoolsAndDemographics(List<MrSchool> highSchools, Map<String, String> gaDemographics, List<SysDepart> departs,
302 List<SpecialStudentVo> specialStudentVos, List<String> mzUnsupporteds, List<String> mzLeftBehinds, 312 List<SpecialStudentVo> specialStudentVos, List<String> mzUnsupporteds, List<String> mzLeftBehinds,
303 List<String> mzAdoptions, Map<String, Family> gaHouseholds, Map<String, GaPunishVo> gaPunishVos, List<String> mzOrphans 313 List<String> mzAdoptions, Map<String, Family> gaHouseholds, Map<String, GaPunishVo> gaPunishVos, List<String> mzOrphans
304 - ) { 314 + ) {
305 try { 315 try {
306 - System.out.println("判断初高中学生信息和人口基本信息的交集和差集zongsuo"+highSchools.size()); 316 + log.info("判断初高中学生信息和人口基本信息的交集和差集zongsuo" + highSchools.size());
307 long startTime = System.currentTimeMillis(); 317 long startTime = System.currentTimeMillis();
308 //不在人口基本信息的初高中学生信息 318 //不在人口基本信息的初高中学生信息
309 List<MrSchool> highSchoolDifference = new ArrayList<>(); 319 List<MrSchool> highSchoolDifference = new ArrayList<>();
@@ -327,10 +337,10 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -327,10 +337,10 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
327 } 337 }
328 }); 338 });
329 } 339 }
330 - System.out.println("初高中交集数"+highSchoolIntersection.size());  
331 - System.out.println("不在初高中学生信息里的的人口基本信息"+gaDemographicDifference.size());  
332 - System.out.println("不在人口基本信息的初高中学生信息"+highSchoolDifference.size());  
333 - doSchoolDifference(highSchoolDifference, departs, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans,gaHouseholds); 340 + log.info("初高中交集数" + highSchoolIntersection.size());
  341 + log.info("不在初高中学生信息里的的人口基本信息" + gaDemographicDifference.size());
  342 + log.info("不在人口基本信息的初高中学生信息" + highSchoolDifference.size());
  343 + doSchoolDifference(highSchoolDifference, departs, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans, gaHouseholds);
334 doIntersection(highSchoolIntersection, departs, gaHouseholds, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans); 344 doIntersection(highSchoolIntersection, departs, gaHouseholds, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans);
335 doDemographicDifference(gaDemographicDifference, departs, gaHouseholds, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans); 345 doDemographicDifference(gaDemographicDifference, departs, gaHouseholds, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans);
336 } catch (Exception e) { 346 } catch (Exception e) {
@@ -345,9 +355,9 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -345,9 +355,9 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
345 public void judgePrimarySchoolsAndDemographics(List<MrSchool> primarySchools, Map<String, String> gaDemographics, List<SysDepart> departs, 355 public void judgePrimarySchoolsAndDemographics(List<MrSchool> primarySchools, Map<String, String> gaDemographics, List<SysDepart> departs,
346 List<SpecialStudentVo> specialStudentVos, List<String> mzUnsupporteds, List<String> mzLeftBehinds, 356 List<SpecialStudentVo> specialStudentVos, List<String> mzUnsupporteds, List<String> mzLeftBehinds,
347 List<String> mzAdoptions, Map<String, Family> gaHouseholds, Map<String, GaPunishVo> gaPunishVos, List<String> mzOrphans 357 List<String> mzAdoptions, Map<String, Family> gaHouseholds, Map<String, GaPunishVo> gaPunishVos, List<String> mzOrphans
348 - ) { 358 + ) {
349 try { 359 try {
350 - System.out.println("判断幼小学生信息和人口基本信息的交集和差集总数"+primarySchools.size()); 360 + log.info("判断幼小学生信息和人口基本信息的交集和差集总数" + primarySchools.size());
351 long startTime = System.currentTimeMillis(); 361 long startTime = System.currentTimeMillis();
352 //不在人口基本信息的幼小学生信息 362 //不在人口基本信息的幼小学生信息
353 List<MrSchool> primarySchoolDifference = new ArrayList<>(); 363 List<MrSchool> primarySchoolDifference = new ArrayList<>();
@@ -371,10 +381,10 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -371,10 +381,10 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
371 } 381 }
372 }); 382 });
373 } 383 }
374 - System.out.println("不在人口基本信息的幼小学生信息"+primarySchoolDifference.size());  
375 - System.out.println("不在幼小学生信息里的的人口基本信息"+gaDemographicDifference.size());  
376 - System.out.println("幼小学生信息和人口基本信息的交集"+primarySchoolIntersection.size());  
377 - doSchoolDifference(primarySchoolDifference, departs, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans,gaHouseholds); 384 + log.info("不在人口基本信息的幼小学生信息" + primarySchoolDifference.size());
  385 + log.info("不在幼小学生信息里的的人口基本信息" + gaDemographicDifference.size());
  386 + log.info("幼小学生信息和人口基本信息的交集" + primarySchoolIntersection.size());
  387 + doSchoolDifference(primarySchoolDifference, departs, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans, gaHouseholds);
378 doIntersection(primarySchoolIntersection, departs, gaHouseholds, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans); 388 doIntersection(primarySchoolIntersection, departs, gaHouseholds, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans);
379 doDemographicDifference(gaDemographicDifference, departs, gaHouseholds, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans); 389 doDemographicDifference(gaDemographicDifference, departs, gaHouseholds, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans);
380 } catch (Exception e) { 390 } catch (Exception e) {
@@ -389,7 +399,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -389,7 +399,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
389 List<String> mzOrphans) { 399 List<String> mzOrphans) {
390 try { 400 try {
391 long startTime = System.currentTimeMillis(); 401 long startTime = System.currentTimeMillis();
392 - System.out.println("处理有人口基本信息但没有学籍信息的数据开始***********************" + gaDemographicDifference.size()); 402 + log.info("处理有人口基本信息但没有学籍信息的数据开始***********************" + gaDemographicDifference.size());
393 Map<String, Minor> minorMap = new HashMap<>(); 403 Map<String, Minor> minorMap = new HashMap<>();
394 if (oConvertUtils.isNotEmpty(gaDemographicDifference) && oConvertUtils.isNotEmpty(gaHouseholds)) { 404 if (oConvertUtils.isNotEmpty(gaDemographicDifference) && oConvertUtils.isNotEmpty(gaHouseholds)) {
395 gaDemographicDifference.stream().forEach(idCard -> { 405 gaDemographicDifference.stream().forEach(idCard -> {
@@ -407,7 +417,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -407,7 +417,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
407 } 417 }
408 }); 418 });
409 } 419 }
410 - System.out.println("处理有人口基本信息但没有学籍信息的数据的所属单位开始" + (System.currentTimeMillis() - startTime) / 1000); 420 + log.info("处理有人口基本信息但没有学籍信息的数据的所属单位开始" + (System.currentTimeMillis() - startTime) / 1000);
411 if (oConvertUtils.isNotEmpty(minorMap) && oConvertUtils.isNotEmpty(departs) && departs.size() > 0) { 421 if (oConvertUtils.isNotEmpty(minorMap) && oConvertUtils.isNotEmpty(departs) && departs.size() > 0) {
412 for (SysDepart d : departs) { 422 for (SysDepart d : departs) {
413 for (Map.Entry<String, Minor> ga : minorMap.entrySet()) { 423 for (Map.Entry<String, Minor> ga : minorMap.entrySet()) {
@@ -420,9 +430,9 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -420,9 +430,9 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
420 } 430 }
421 } 431 }
422 } 432 }
423 - System.out.println("处理有人口基本信息但没有学籍信息的数据的所属单位结束" + (System.currentTimeMillis() - startTime) / 1000);  
424 - System.out.println("处理有人口基本信息但没有学籍信息的数据jieshu**********************" + minorMap.size());  
425 - getSpecialStudent(minorMap, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans,gaHouseholds); 433 + log.info("处理有人口基本信息但没有学籍信息的数据的所属单位结束" + (System.currentTimeMillis() - startTime) / 1000);
  434 + log.info("处理有人口基本信息但没有学籍信息的数据jieshu**********************" + minorMap.size());
  435 + getSpecialStudent(minorMap, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans, gaHouseholds);
426 } catch (Exception e) { 436 } catch (Exception e) {
427 e.printStackTrace(); 437 e.printStackTrace();
428 log.info(String.valueOf(e)); 438 log.info(String.valueOf(e));
@@ -436,29 +446,29 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -436,29 +446,29 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
436 long startTime = System.currentTimeMillis(); 446 long startTime = System.currentTimeMillis();
437 try { 447 try {
438 Map<String, Minor> minorMap = new HashMap<>(); 448 Map<String, Minor> minorMap = new HashMap<>();
439 - Map<String, MrSchool> schools=new HashMap<>();  
440 - System.out.println("***********************交集总数**********************************************" + intersection.size());  
441 - System.out.println("***********************人口信息总数**********************************************" + gaHouseholds.size());  
442 - System.out.println("交集数据里的所属单位开始" + (System.currentTimeMillis() - startTime) / 1000); 449 + Map<String, MrSchool> schools = new HashMap<>();
  450 + log.info("***********************交集总数**********************************************" + intersection.size());
  451 + log.info("***********************人口信息总数**********************************************" + gaHouseholds.size());
  452 + log.info("交集数据里的所属单位开始" + (System.currentTimeMillis() - startTime) / 1000);
443 Date creatDate = new Date(); 453 Date creatDate = new Date();
444 if (oConvertUtils.isNotEmpty(intersection) && oConvertUtils.isNotEmpty(departs) && departs.size() > 0) { 454 if (oConvertUtils.isNotEmpty(intersection) && oConvertUtils.isNotEmpty(departs) && departs.size() > 0) {
445 for (SysDepart d : departs) { 455 for (SysDepart d : departs) {
446 intersection.stream().forEach(i -> { 456 intersection.stream().forEach(i -> {
447 Minor minor = new Minor(); 457 Minor minor = new Minor();
448 - if (oConvertUtils.isNotEmpty(i) && oConvertUtils.isNotEmpty(i.getIdentity()) && IDNumberUtil.checkID(i.getIdentity())) {  
449 - String idCard=i.getIdentity();  
450 - if(oConvertUtils.isNotEmpty(minorMap) && oConvertUtils.isNotEmpty(minorMap.get(idCard))){  
451 - Minor minMap=minorMap.get(idCard);  
452 - if(oConvertUtils.isNotEmpty(d) && oConvertUtils.isNotEmpty(d.getCommonCode()) && oConvertUtils.isNotEmpty(minMap.getSchoolName())){ 458 + if (oConvertUtils.isNotEmpty(i) && oConvertUtils.isNotEmpty(i.getIdentity()) && IDNumberUtil.checkID(i.getIdentity())) {
  459 + String idCard = i.getIdentity();
  460 + if (oConvertUtils.isNotEmpty(minorMap) && oConvertUtils.isNotEmpty(minorMap.get(idCard))) {
  461 + Minor minMap = minorMap.get(idCard);
  462 + if (oConvertUtils.isNotEmpty(d) && oConvertUtils.isNotEmpty(d.getCommonCode()) && oConvertUtils.isNotEmpty(minMap.getSchoolName())) {
453 if (d.getCommonCode().contains(minMap.getSchoolName())) { 463 if (d.getCommonCode().contains(minMap.getSchoolName())) {
454 minMap.setSysOrgCode(d.getOrgCode()); 464 minMap.setSysOrgCode(d.getOrgCode());
455 i.setSysOrgCode(d.getOrgCode()); 465 i.setSysOrgCode(d.getOrgCode());
456 } 466 }
457 } 467 }
458 - if(oConvertUtils.isNotEmpty(schools) && oConvertUtils.isNotEmpty(schools.get(idCard)) ){ 468 + if (oConvertUtils.isNotEmpty(schools) && oConvertUtils.isNotEmpty(schools.get(idCard))) {
459 schools.get(idCard).setSysOrgCode(i.getSysOrgCode()); 469 schools.get(idCard).setSysOrgCode(i.getSysOrgCode());
460 } 470 }
461 - }else { 471 + } else {
462 if (oConvertUtils.isNotEmpty(d) && oConvertUtils.isNotEmpty(d.getCommonCode()) && oConvertUtils.isNotEmpty(i.getSchool())) { 472 if (oConvertUtils.isNotEmpty(d) && oConvertUtils.isNotEmpty(d.getCommonCode()) && oConvertUtils.isNotEmpty(i.getSchool())) {
463 if (d.getCommonCode().contains(i.getSchool())) { 473 if (d.getCommonCode().contains(i.getSchool())) {
464 minor.setSysOrgCode(d.getOrgCode()); 474 minor.setSysOrgCode(d.getOrgCode());
@@ -493,12 +503,12 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -493,12 +503,12 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
493 }); 503 });
494 } 504 }
495 } 505 }
496 - System.out.println("交集数据里的所属单位结束总数"+intersection.size()+"交集数据里的所属单位结束" + (System.currentTimeMillis() - startTime) / 1000);  
497 - System.out.println("户籍信息里的未成年人的单位" + (System.currentTimeMillis() - startTime) / 1000 + "秒"); 506 + log.info("交集数据里的所属单位结束总数" + intersection.size() + "交集数据里的所属单位结束" + (System.currentTimeMillis() - startTime) / 1000);
  507 + log.info("户籍信息里的未成年人的单位" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
498 long startTime1 = System.currentTimeMillis(); 508 long startTime1 = System.currentTimeMillis();
499 resultSchools.addAll(schools.values()); 509 resultSchools.addAll(schools.values());
500 - System.out.println("人口信息he交集总数总数" + minorMap.size());  
501 - getSpecialStudent(minorMap, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans,gaHouseholds); 510 + log.info("人口信息he交集总数总数" + minorMap.size());
  511 + getSpecialStudent(minorMap, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans, gaHouseholds);
502 } catch (Exception e) { 512 } catch (Exception e) {
503 e.printStackTrace(); 513 e.printStackTrace();
504 log.info(String.valueOf(e)); 514 log.info(String.valueOf(e));
@@ -511,29 +521,29 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -511,29 +521,29 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
511 List<String> mzOrphans, Map<String, Family> gaHouseholds) { 521 List<String> mzOrphans, Map<String, Family> gaHouseholds) {
512 try { 522 try {
513 Map<String, Minor> minorMap = new HashMap<>(); 523 Map<String, Minor> minorMap = new HashMap<>();
514 - Map<String, MrSchool> schools=new HashMap<>(); 524 + Map<String, MrSchool> schools = new HashMap<>();
515 long startTime = System.currentTimeMillis(); 525 long startTime = System.currentTimeMillis();
516 - System.out.println("根据差集数据判断所属单位" + difference.size());  
517 - System.out.println("未成年人总数" + difference.size()); 526 + log.info("根据差集数据判断所属单位" + difference.size());
  527 + log.info("未成年人总数" + difference.size());
518 Date creatTime = new Date(); 528 Date creatTime = new Date();
519 if (oConvertUtils.isNotEmpty(difference) && oConvertUtils.isNotEmpty(departs)) { 529 if (oConvertUtils.isNotEmpty(difference) && oConvertUtils.isNotEmpty(departs)) {
520 for (SysDepart d : departs) { 530 for (SysDepart d : departs) {
521 difference.stream().forEach(m -> { 531 difference.stream().forEach(m -> {
522 Minor minor = new Minor(); 532 Minor minor = new Minor();
523 - if (oConvertUtils.isNotEmpty(m) && oConvertUtils.isNotEmpty(m.getIdentity()) && IDNumberUtil.checkID(m.getIdentity())) {  
524 - String idCard=m.getIdentity();  
525 - if(oConvertUtils.isNotEmpty(minorMap) && oConvertUtils.isNotEmpty(minorMap.get(idCard))){  
526 - Minor minMap=minorMap.get(idCard);  
527 - if(oConvertUtils.isNotEmpty(d) && oConvertUtils.isNotEmpty(d.getCommonCode()) && oConvertUtils.isNotEmpty(minMap.getSchoolName())){  
528 - if (d.getCommonCode().contains(minMap.getSchoolName())) {  
529 - minMap.setSysOrgCode(d.getOrgCode());  
530 - m.setSysOrgCode(d.getOrgCode());  
531 - }  
532 - }  
533 - if(oConvertUtils.isNotEmpty(schools) && oConvertUtils.isNotEmpty(schools.get(idCard)) ){ 533 + if (oConvertUtils.isNotEmpty(m) && oConvertUtils.isNotEmpty(m.getIdentity()) && IDNumberUtil.checkID(m.getIdentity())) {
  534 + String idCard = m.getIdentity();
  535 + if (oConvertUtils.isNotEmpty(minorMap) && oConvertUtils.isNotEmpty(minorMap.get(idCard))) {
  536 + Minor minMap = minorMap.get(idCard);
  537 + if (oConvertUtils.isNotEmpty(d) && oConvertUtils.isNotEmpty(d.getCommonCode()) && oConvertUtils.isNotEmpty(minMap.getSchoolName())) {
  538 + if (d.getCommonCode().contains(minMap.getSchoolName())) {
  539 + minMap.setSysOrgCode(d.getOrgCode());
  540 + m.setSysOrgCode(d.getOrgCode());
  541 + }
  542 + }
  543 + if (oConvertUtils.isNotEmpty(schools) && oConvertUtils.isNotEmpty(schools.get(idCard))) {
534 schools.get(idCard).setSysOrgCode(m.getSysOrgCode()); 544 schools.get(idCard).setSysOrgCode(m.getSysOrgCode());
535 } 545 }
536 - }else { 546 + } else {
537 if (oConvertUtils.isNotEmpty(d) && oConvertUtils.isNotEmpty(d.getCommonCode()) && oConvertUtils.isNotEmpty(m.getSchool())) { 547 if (oConvertUtils.isNotEmpty(d) && oConvertUtils.isNotEmpty(d.getCommonCode()) && oConvertUtils.isNotEmpty(m.getSchool())) {
538 if (d.getCommonCode().contains(m.getSchool())) { 548 if (d.getCommonCode().contains(m.getSchool())) {
539 minor.setSysOrgCode(d.getOrgCode()); 549 minor.setSysOrgCode(d.getOrgCode());
@@ -569,12 +579,12 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -569,12 +579,12 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
569 }); 579 });
570 } 580 }
571 } 581 }
572 - System.out.println("根据不在人口基础信息里的学生和单位信息判断所属单位" + (System.currentTimeMillis() - startTime) / 1000 + "秒"); 582 + log.info("根据不在人口基础信息里的学生和单位信息判断所属单位" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
573 long startTime1 = System.currentTimeMillis(); 583 long startTime1 = System.currentTimeMillis();
574 // schoolService.insertBatch(difference); 584 // schoolService.insertBatch(difference);
575 resultSchools.addAll(schools.values()); 585 resultSchools.addAll(schools.values());
576 - System.out.println("有学籍信息但没有人口基本信息的数据" + minorMap.size());  
577 - getSpecialStudent(minorMap, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans,gaHouseholds); 586 + log.info("有学籍信息但没有人口基本信息的数据" + minorMap.size());
  587 + getSpecialStudent(minorMap, specialStudentVos, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans, gaHouseholds);
578 } catch (Exception e) { 588 } catch (Exception e) {
579 e.printStackTrace(); 589 e.printStackTrace();
580 log.info(String.valueOf(e)); 590 log.info(String.valueOf(e));
@@ -591,7 +601,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -591,7 +601,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
591 List<String> mzAdoptions, Map<String, GaPunishVo> gaPunishVos, List<String> mzOrphans, Map<String, Family> gaHouseholds) { 601 List<String> mzAdoptions, Map<String, GaPunishVo> gaPunishVos, List<String> mzOrphans, Map<String, Family> gaHouseholds) {
592 long startTime = System.currentTimeMillis(); 602 long startTime = System.currentTimeMillis();
593 try { 603 try {
594 - System.out.println("根据特殊学校信息判断未成年人重点关注原因和备注"); 604 + log.info("根据特殊学校信息判断未成年人重点关注原因和备注");
595 if (oConvertUtils.isNotEmpty(specialStudentVos) && oConvertUtils.isNotEmpty(minors)) { 605 if (oConvertUtils.isNotEmpty(specialStudentVos) && oConvertUtils.isNotEmpty(minors)) {
596 for (SpecialStudentVo s : specialStudentVos) { 606 for (SpecialStudentVo s : specialStudentVos) {
597 if (oConvertUtils.isNotEmpty(s.getIdentity()) && oConvertUtils.isNotEmpty(minors.get(s.getIdentity()))) { 607 if (oConvertUtils.isNotEmpty(s.getIdentity()) && oConvertUtils.isNotEmpty(minors.get(s.getIdentity()))) {
@@ -601,8 +611,8 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -601,8 +611,8 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
601 } 611 }
602 } 612 }
603 } 613 }
604 - System.out.println("根据特殊学校信息判断未成年人重点关注原因和备注" + (System.currentTimeMillis() - startTime) / 1000 + "秒");  
605 - getUnsupported(minors, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans,gaHouseholds); 614 + log.info("根据特殊学校信息判断未成年人重点关注原因和备注" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
  615 + getUnsupported(minors, mzUnsupporteds, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans, gaHouseholds);
606 } catch (Exception e) { 616 } catch (Exception e) {
607 e.printStackTrace(); 617 e.printStackTrace();
608 log.info(String.valueOf(e)); 618 log.info(String.valueOf(e));
@@ -617,21 +627,21 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -617,21 +627,21 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
617 List<String> mzAdoptions, Map<String, GaPunishVo> gaPunishVos, List<String> mzOrphans, Map<String, Family> gaHouseholds) { 627 List<String> mzAdoptions, Map<String, GaPunishVo> gaPunishVos, List<String> mzOrphans, Map<String, Family> gaHouseholds) {
618 long startTime = System.currentTimeMillis(); 628 long startTime = System.currentTimeMillis();
619 try { 629 try {
620 - System.out.println("无人抚养");  
621 - String reason="无人抚养"; 630 + log.info("无人抚养");
  631 + String reason = "无人抚养";
622 if (oConvertUtils.isNotEmpty(minors) && oConvertUtils.isNotEmpty(mzUnsupporteds)) { 632 if (oConvertUtils.isNotEmpty(minors) && oConvertUtils.isNotEmpty(mzUnsupporteds)) {
623 for (String s : mzUnsupporteds) { 633 for (String s : mzUnsupporteds) {
624 - if (oConvertUtils.isNotEmpty(s) && oConvertUtils.isNotEmpty(minors.get(s)) ) {  
625 - if(oConvertUtils.isNotEmpty(minors.get(s).getReason()) && !minors.get(s).getReason().contains(reason)){  
626 - minors.get(s).setReason(minors.get(s).getReason()+","+reason);  
627 - }else { 634 + if (oConvertUtils.isNotEmpty(s) && oConvertUtils.isNotEmpty(minors.get(s))) {
  635 + if (oConvertUtils.isNotEmpty(minors.get(s).getReason()) && !minors.get(s).getReason().contains(reason)) {
  636 + minors.get(s).setReason(minors.get(s).getReason() + "," + reason);
  637 + } else {
628 minors.get(s).setReason(reason); 638 minors.get(s).setReason(reason);
629 } 639 }
630 } 640 }
631 } 641 }
632 } 642 }
633 - System.out.println("无人抚养" + (System.currentTimeMillis() - startTime) / 1000 + "秒");  
634 - getAdoption(minors, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans,gaHouseholds); 643 + log.info("无人抚养" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
  644 + getAdoption(minors, mzLeftBehinds, mzAdoptions, gaPunishVos, mzOrphans, gaHouseholds);
635 } catch (Exception e) { 645 } catch (Exception e) {
636 e.printStackTrace(); 646 e.printStackTrace();
637 log.info(String.valueOf(e)); 647 log.info(String.valueOf(e));
@@ -645,21 +655,21 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -645,21 +655,21 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
645 public void getAdoption(Map<String, Minor> minors, List<String> mzLeftBehinds, List<String> mzAdoptions, Map<String, GaPunishVo> gaPunishVos, List<String> mzOrphans, Map<String, Family> gaHouseholds) { 655 public void getAdoption(Map<String, Minor> minors, List<String> mzLeftBehinds, List<String> mzAdoptions, Map<String, GaPunishVo> gaPunishVos, List<String> mzOrphans, Map<String, Family> gaHouseholds) {
646 long startTime = System.currentTimeMillis(); 656 long startTime = System.currentTimeMillis();
647 try { 657 try {
648 - System.out.println("收养");  
649 - String reason="收养"; 658 + log.info("收养");
  659 + String reason = "收养";
650 if (oConvertUtils.isNotEmpty(minors) && oConvertUtils.isNotEmpty(mzAdoptions)) { 660 if (oConvertUtils.isNotEmpty(minors) && oConvertUtils.isNotEmpty(mzAdoptions)) {
651 for (String s : mzAdoptions) { 661 for (String s : mzAdoptions) {
652 - if (oConvertUtils.isNotEmpty(s) && oConvertUtils.isNotEmpty(minors.get(s)) ) {  
653 - if(oConvertUtils.isNotEmpty(minors.get(s).getReason()) && !minors.get(s).getReason().contains(reason)){  
654 - minors.get(s).setReason(minors.get(s).getReason()+","+reason);  
655 - }else { 662 + if (oConvertUtils.isNotEmpty(s) && oConvertUtils.isNotEmpty(minors.get(s))) {
  663 + if (oConvertUtils.isNotEmpty(minors.get(s).getReason()) && !minors.get(s).getReason().contains(reason)) {
  664 + minors.get(s).setReason(minors.get(s).getReason() + "," + reason);
  665 + } else {
656 minors.get(s).setReason(reason); 666 minors.get(s).setReason(reason);
657 } 667 }
658 } 668 }
659 } 669 }
660 } 670 }
661 - System.out.println("收养" + (System.currentTimeMillis() - startTime) / 1000 + "秒");  
662 - getChildren(minors, mzLeftBehinds, gaPunishVos, mzOrphans,gaHouseholds); 671 + log.info("收养" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
  672 + getChildren(minors, mzLeftBehinds, gaPunishVos, mzOrphans, gaHouseholds);
663 } catch (Exception e) { 673 } catch (Exception e) {
664 e.printStackTrace(); 674 e.printStackTrace();
665 log.info(String.valueOf(e)); 675 log.info(String.valueOf(e));
@@ -673,21 +683,21 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -673,21 +683,21 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
673 public void getChildren(Map<String, Minor> minors, List<String> mzLeftBehinds, Map<String, GaPunishVo> gaPunishVos, List<String> mzOrphans, Map<String, Family> gaHouseholds) { 683 public void getChildren(Map<String, Minor> minors, List<String> mzLeftBehinds, Map<String, GaPunishVo> gaPunishVos, List<String> mzOrphans, Map<String, Family> gaHouseholds) {
674 long startTime = System.currentTimeMillis(); 684 long startTime = System.currentTimeMillis();
675 try { 685 try {
676 - System.out.println("留守儿童");  
677 - String reason="留守儿童"; 686 + log.info("留守儿童");
  687 + String reason = "留守儿童";
678 if (oConvertUtils.isNotEmpty(minors) && oConvertUtils.isNotEmpty(mzLeftBehinds)) { 688 if (oConvertUtils.isNotEmpty(minors) && oConvertUtils.isNotEmpty(mzLeftBehinds)) {
679 for (String s : mzLeftBehinds) { 689 for (String s : mzLeftBehinds) {
680 if (oConvertUtils.isNotEmpty(s) && oConvertUtils.isNotEmpty(minors.get(s))) { 690 if (oConvertUtils.isNotEmpty(s) && oConvertUtils.isNotEmpty(minors.get(s))) {
681 - if(oConvertUtils.isNotEmpty(minors.get(s).getReason()) && !minors.get(s).getReason().contains(reason)){  
682 - minors.get(s).setReason(minors.get(s).getReason()+","+reason);  
683 - }else { 691 + if (oConvertUtils.isNotEmpty(minors.get(s).getReason()) && !minors.get(s).getReason().contains(reason)) {
  692 + minors.get(s).setReason(minors.get(s).getReason() + "," + reason);
  693 + } else {
684 minors.get(s).setReason(reason); 694 minors.get(s).setReason(reason);
685 } 695 }
686 } 696 }
687 } 697 }
688 } 698 }
689 - System.out.println("留守儿童" + (System.currentTimeMillis() - startTime) / 1000 + "秒");  
690 - getOrphan(minors, mzOrphans, gaPunishVos,gaHouseholds); 699 + log.info("留守儿童" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
  700 + getOrphan(minors, mzOrphans, gaPunishVos, gaHouseholds);
691 } catch (Exception e) { 701 } catch (Exception e) {
692 e.printStackTrace(); 702 e.printStackTrace();
693 log.info(String.valueOf(e)); 703 log.info(String.valueOf(e));
@@ -701,21 +711,21 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -701,21 +711,21 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
701 public void getOrphan(Map<String, Minor> minors, List<String> mzOrphans, Map<String, GaPunishVo> gaPunishVos, Map<String, Family> gaHouseholds) { 711 public void getOrphan(Map<String, Minor> minors, List<String> mzOrphans, Map<String, GaPunishVo> gaPunishVos, Map<String, Family> gaHouseholds) {
702 long startTime = System.currentTimeMillis(); 712 long startTime = System.currentTimeMillis();
703 try { 713 try {
704 - System.out.println("孤儿");  
705 - String reason="孤儿"; 714 + log.info("孤儿");
  715 + String reason = "孤儿";
706 if (oConvertUtils.isNotEmpty(minors) && oConvertUtils.isNotEmpty(mzOrphans)) { 716 if (oConvertUtils.isNotEmpty(minors) && oConvertUtils.isNotEmpty(mzOrphans)) {
707 for (String s : mzOrphans) { 717 for (String s : mzOrphans) {
708 - if (oConvertUtils.isNotEmpty(s) && oConvertUtils.isNotEmpty(minors.get(s)) ) {  
709 - if(oConvertUtils.isNotEmpty(minors.get(s).getReason()) && !minors.get(s).getReason().contains(reason)){  
710 - minors.get(s).setReason(minors.get(s).getReason()+","+reason);  
711 - }else { 718 + if (oConvertUtils.isNotEmpty(s) && oConvertUtils.isNotEmpty(minors.get(s))) {
  719 + if (oConvertUtils.isNotEmpty(minors.get(s).getReason()) && !minors.get(s).getReason().contains(reason)) {
  720 + minors.get(s).setReason(minors.get(s).getReason() + "," + reason);
  721 + } else {
712 minors.get(s).setReason(reason); 722 minors.get(s).setReason(reason);
713 } 723 }
714 } 724 }
715 } 725 }
716 } 726 }
717 - System.out.println("孤儿" + (System.currentTimeMillis() - startTime) / 1000 + "秒");  
718 - getFamilyRelationship(minors, gaHouseholds,gaPunishVos); 727 + log.info("孤儿" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
  728 + getFamilyRelationship(minors, gaHouseholds, gaPunishVos);
719 } catch (Exception e) { 729 } catch (Exception e) {
720 e.printStackTrace(); 730 e.printStackTrace();
721 log.info(String.valueOf(e)); 731 log.info(String.valueOf(e));
@@ -758,7 +768,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -758,7 +768,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
758 } else { 768 } else {
759 family.setCrime(Family.IS_NOT_CRIME); 769 family.setCrime(Family.IS_NOT_CRIME);
760 } 770 }
761 - if(IDNumberUtil.checkID(mapKey)){ 771 + if (IDNumberUtil.checkID(mapKey)) {
762 family.setGender(Integer.valueOf(IDNumberUtil.judgeGender(mapKey))); 772 family.setGender(Integer.valueOf(IDNumberUtil.judgeGender(mapKey)));
763 } 773 }
764 family.setId(String.valueOf(id)); 774 family.setId(String.valueOf(id));
@@ -777,7 +787,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -777,7 +787,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
777 familyMaps.put(houseNum, list); 787 familyMaps.put(houseNum, list);
778 } 788 }
779 if (oConvertUtils.isNotEmpty(family.getCrime()) && family.getCrime().equals(Family.IS_CRIME)) { 789 if (oConvertUtils.isNotEmpty(family.getCrime()) && family.getCrime().equals(Family.IS_CRIME)) {
780 - idCards.put(houseNum,true); 790 + idCards.put(houseNum, true);
781 } 791 }
782 if (oConvertUtils.isNotEmpty(family.getRelation()) && family.getRelation().equals("户主")) { 792 if (oConvertUtils.isNotEmpty(family.getRelation()) && family.getRelation().equals("户主")) {
783 guardians.put(houseNum, family); 793 guardians.put(houseNum, family);
@@ -785,7 +795,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -785,7 +795,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
785 } 795 }
786 } 796 }
787 } 797 }
788 - System.out.println("建立未成年人和家庭成员关系"); 798 + log.info("建立未成年人和家庭成员关系");
789 getFamilyCrime(minors, idCards, familyMaps, guardians, relations); 799 getFamilyCrime(minors, idCards, familyMaps, guardians, relations);
790 } catch (Exception e) { 800 } catch (Exception e) {
791 e.printStackTrace(); 801 e.printStackTrace();
@@ -804,11 +814,11 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -804,11 +814,11 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
804 long startTime = System.currentTimeMillis(); 814 long startTime = System.currentTimeMillis();
805 try { 815 try {
806 LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal(); 816 LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
807 - System.out.println("家庭成员是否犯罪"); 817 + log.info("家庭成员是否犯罪");
808 List<Minor> minorList = new ArrayList<>(); 818 List<Minor> minorList = new ArrayList<>();
809 Set<Family> familys = new HashSet<>(); 819 Set<Family> familys = new HashSet<>();
810 //存放已经被放入minor里面的户号，避免数据重复 820 //存放已经被放入minor里面的户号，避免数据重复
811 - Set<String> houseSet=new HashSet<>(); 821 + Set<String> houseSet = new HashSet<>();
812 Date createTime = new Date(); 822 Date createTime = new Date();
813 if (oConvertUtils.isNotEmpty(minors)) { 823 if (oConvertUtils.isNotEmpty(minors)) {
814 for (Map.Entry<String, Minor> m : minors.entrySet()) { 824 for (Map.Entry<String, Minor> m : minors.entrySet()) {
@@ -818,12 +828,12 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -818,12 +828,12 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
818 if (oConvertUtils.isNotEmpty(user) && oConvertUtils.isNotEmpty(user.getId())) { 828 if (oConvertUtils.isNotEmpty(user) && oConvertUtils.isNotEmpty(user.getId())) {
819 minor.setCreateBy(user.getId()); 829 minor.setCreateBy(user.getId());
820 } 830 }
821 - List<Family> f=new ArrayList<>(); 831 + List<Family> f = new ArrayList<>();
822 //判断是否是一个家庭的人员 832 //判断是否是一个家庭的人员
823 if (oConvertUtils.isNotEmpty(minor.getHouseholdNum())) { 833 if (oConvertUtils.isNotEmpty(minor.getHouseholdNum())) {
824 String houseNum = minor.getHouseholdNum(); 834 String houseNum = minor.getHouseholdNum();
825 - if(oConvertUtils.isNotEmpty(familyMaps) && oConvertUtils.isNotEmpty(familyMaps.get(houseNum)) ){  
826 - if(!testIdCards.contains(houseNum) ){ 835 + if (oConvertUtils.isNotEmpty(familyMaps) && oConvertUtils.isNotEmpty(familyMaps.get(houseNum))) {
  836 + if (!testIdCards.contains(houseNum)) {
827 f.addAll(familyMaps.get(houseNum)); 837 f.addAll(familyMaps.get(houseNum));
828 // resultFamily.addAll(familyMaps.get(houseNum)); 838 // resultFamily.addAll(familyMaps.get(houseNum));
829 testIdCards.add(houseNum); 839 testIdCards.add(houseNum);
@@ -834,12 +844,12 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -834,12 +844,12 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
834 if (oConvertUtils.isNotEmpty(houseNum) && oConvertUtils.isNotEmpty(guardians) && oConvertUtils.isNotEmpty(guardians.get(houseNum))) { 844 if (oConvertUtils.isNotEmpty(houseNum) && oConvertUtils.isNotEmpty(guardians) && oConvertUtils.isNotEmpty(guardians.get(houseNum))) {
835 minor.setGuardian(guardians.get(houseNum).getName()); 845 minor.setGuardian(guardians.get(houseNum).getName());
836 } 846 }
837 - String reason="家庭成员有犯罪记录"; 847 + String reason = "家庭成员有犯罪记录";
838 //判断家庭成员是否犯罪 848 //判断家庭成员是否犯罪
839 if (oConvertUtils.isNotEmpty(idCard) && oConvertUtils.isNotEmpty(idCards) && oConvertUtils.isNotEmpty(idCards.get(houseNum))) { 849 if (oConvertUtils.isNotEmpty(idCard) && oConvertUtils.isNotEmpty(idCards) && oConvertUtils.isNotEmpty(idCards.get(houseNum))) {
840 - if(oConvertUtils.isNotEmpty(minor.getReason()) && !minor.getReason().contains(reason)){  
841 - minor.setReason(minor.getReason()+","+reason);  
842 - }else { 850 + if (oConvertUtils.isNotEmpty(minor.getReason()) && !minor.getReason().contains(reason)) {
  851 + minor.setReason(minor.getReason() + "," + reason);
  852 + } else {
843 minor.setReason(reason); 853 minor.setReason(reason);
844 } 854 }
845 } 855 }
@@ -855,7 +865,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra @@ -855,7 +865,7 @@ public class AnalyzeController extends JeecgController&lt;GaDemographic, IGaDemogra
855 e.printStackTrace(); 865 e.printStackTrace();
856 log.info(String.valueOf(e)); 866 log.info(String.valueOf(e));
857 } 867 }
858 - System.out.println("家庭成员是否犯罪" + (System.currentTimeMillis() - startTime) / 1000 + "秒"); 868 + log.info("家庭成员是否犯罪" + (System.currentTimeMillis() - startTime) / 1000 + "秒");
859 869
860 } 870 }
861 871
juvenile-prosecution-boot/jeecg-boot-module-system/src/main/resources/application-dev.yml
  View file @7f4d250
@@ -311,4 +311,12 @@ third-app: @@ -311,4 +311,12 @@ third-app:
311 client-id: ?? 311 client-id: ??
312 # appSecret 312 # appSecret
313 client-secret: ?? 313 client-secret: ??
314 - agent-id: ??  
315 \ No newline at end of file 314 \ No newline at end of file
  315 + agent-id: ??
  316 +#Spark配置
  317 +spark:
  318 + datasource:
  319 + url: jdbc:mysql://192.168.1.201:3306/juvenile-prosecution?rewriteBatchedStatements=true&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true
  320 + username: root
  321 + password: mx123456
  322 + driver: com.mysql.cj.jdbc.Driver
  323 + enabled: true
316 \ No newline at end of file 324 \ No newline at end of file