204 changed files with 7052 additions and 8207 deletions
@ -0,0 +1,60 @@ |
|||||
|
package io.sc.platform.core.autoconfigure; |
||||
|
|
||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; |
||||
|
import org.springframework.context.annotation.Bean; |
||||
|
import org.springframework.context.annotation.Configuration; |
||||
|
import org.springframework.http.client.ClientHttpRequestFactory; |
||||
|
import org.springframework.http.client.SimpleClientHttpRequestFactory; |
||||
|
import org.springframework.http.converter.HttpMessageConverter; |
||||
|
import org.springframework.http.converter.StringHttpMessageConverter; |
||||
|
import org.springframework.http.converter.xml.Jaxb2RootElementHttpMessageConverter; |
||||
|
import org.springframework.http.converter.xml.MappingJackson2XmlHttpMessageConverter; |
||||
|
import org.springframework.http.converter.xml.SourceHttpMessageConverter; |
||||
|
import org.springframework.web.client.RestTemplate; |
||||
|
|
||||
|
import java.nio.charset.StandardCharsets; |
||||
|
import java.util.ArrayList; |
||||
|
import java.util.List; |
||||
|
|
||||
|
@Configuration(proxyBeanMethods = false) |
||||
|
public class RestTemplateAutoConfiguration { |
||||
|
@Bean |
||||
|
@ConditionalOnMissingBean |
||||
|
public RestTemplate restTemplate(ClientHttpRequestFactory factory){ |
||||
|
RestTemplate restTemplate =new RestTemplate(factory); |
||||
|
List<HttpMessageConverter<?>> converters =restTemplate.getMessageConverters(); |
||||
|
if(!converters.isEmpty()) { |
||||
|
for(HttpMessageConverter<?> converter : converters) { |
||||
|
if(converter instanceof StringHttpMessageConverter) { |
||||
|
StringHttpMessageConverter stringHttpMessageConverter =(StringHttpMessageConverter)converter; |
||||
|
stringHttpMessageConverter.setDefaultCharset(StandardCharsets.UTF_8); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
//移除 xml 相关的 MessageConverter
|
||||
|
List<HttpMessageConverter<?>> willBeRemoves =new ArrayList<>(); |
||||
|
for(HttpMessageConverter<?> converter : converters) { |
||||
|
if( converter instanceof SourceHttpMessageConverter<?> |
||||
|
|| converter instanceof MappingJackson2XmlHttpMessageConverter |
||||
|
|| converter instanceof Jaxb2RootElementHttpMessageConverter |
||||
|
) { |
||||
|
willBeRemoves.add(converter); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
for(HttpMessageConverter<?> removed : willBeRemoves) { |
||||
|
converters.remove(removed); |
||||
|
} |
||||
|
} |
||||
|
return restTemplate; |
||||
|
} |
||||
|
|
||||
|
@Bean |
||||
|
@ConditionalOnMissingBean |
||||
|
public ClientHttpRequestFactory simpleClientHttpRequestFactory(){ |
||||
|
SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); |
||||
|
factory.setReadTimeout(30000); |
||||
|
factory.setConnectTimeout(5000); |
||||
|
return factory; |
||||
|
} |
||||
|
} |
||||
Binary file not shown.
@ -0,0 +1,740 @@ |
|||||
|
[appendix] |
||||
|
= JAVA |
||||
|
== Java 8 Time Api |
||||
|
Java 8 为 Date 和 Time 引入了新的 API，以解决旧 java.util.Date 和 java.util.Calendar 的缺点。 |
||||
|
|
||||
|
=== 旧的时间API（java8之前）的问题 |
||||
|
* 线程安全 - Date 和 Calendar 类不是线程安全的，使开发者难以调试这些 api 的并发问题，需要编写额外的代码来处理线程安全。Java 8 中引入的新的 Date 和 Time API 是不可变的和线程安全的，使得这些痛点得以解决。 |
||||
|
* API 设计和易于理解 - 旧的时间 api 非常难以理解，操作都非常复杂，非常绕口，没有提供一些常用的解析转换方法。新的时间 API 是以 ISO 为中心的，并遵循 date, time, duration 和 periods 的一致域模型。提供了一些非常实用方法以支持最常见的操作。不再需要我们自己封装一些时间操作类，而且描述语义化。 |
||||
|
* ZonedDate 和 Time - 在旧的时间 api 中开发人员必须编写额外的逻辑来处理旧 API 的时区逻辑，而使用新的 API，可以使用 Local 和 ZonedDate / Time API 来处理时区。无需过多关心时区转换问题。 |
||||
|
|
||||
|
=== 使用 LocalDate，LocalTime 和 LocalDateTime |
||||
|
上下文相结合的本地日期/时间。这些类主要用于不需要在上下文中明确指定时区的情况。 |
||||
|
|
||||
|
==== LocalDate |
||||
|
[source,java] |
||||
|
---- |
||||
|
// 获取当前系统时钟下的日期 |
||||
|
LocalDate localDate = LocalDate.now(); |
||||
|
|
||||
|
// 构建指定日期 |
||||
|
LocalDate localDate =LocalDate.of(2015, 02, 20); |
||||
|
LocalDate localDate =LocalDate.parse("2015-02-20"); |
||||
|
|
||||
|
// 当前日期加一天 |
||||
|
LocalDate tomorrow =LocalDate.now().plusDays(1); |
||||
|
|
||||
|
// 当前日期减一个月 |
||||
|
LocalDate previousMonthSameDay =LocalDate.now().minus(1, ChronoUnit.MONTHS); |
||||
|
|
||||
|
// 星期几 |
||||
|
DayOfWeek sunday =LocalDate.parse("2019-06-12").getDayOfWeek(); |
||||
|
|
||||
|
// 月中的第几天 |
||||
|
int twelve =LocalDate.parse("2016-09-12").getDayOfMonth(); |
||||
|
|
||||
|
// 是否闰年 |
||||
|
boolean leapYear =LocalDate.now().isLeapYear(); |
||||
|
|
||||
|
// 日期先后 |
||||
|
boolean notBefore =LocalDate.parse("2019-06-12").isBefore(LocalDate.parse("2019-06-11")); |
||||
|
boolean isAfter =LocalDate.parse("2019-06-12").isAfter(LocalDate.parse("2019-06-11")); |
||||
|
|
||||
|
// 开始时间 |
||||
|
LocalDateTime beginningOfDay =LocalDate.parse("2019-06-12").atStartOfDay(); |
||||
|
|
||||
|
// 月初 |
||||
|
LocalDate firstDayOfMonth =LocalDate.parse("2019-09-12").with(TemporalAdjusters.firstDayOfMonth()); |
||||
|
---- |
||||
|
|
||||
|
==== LocalTime |
||||
|
[source,java] |
||||
|
---- |
||||
|
// 当前时间 |
||||
|
LocalTime now =LocalTime.now(); |
||||
|
|
||||
|
// 构建时间 |
||||
|
LocalTime sixThirty =LocalTime.parse("06:30"); |
||||
|
LocalTime sixThirty =LocalTime.of(6, 30); |
||||
|
|
||||
|
// 当前时间加一小时 |
||||
|
LocalTime sevenThirty =LocalTime.parse("06:30").plus(1, ChronoUnit.HOURS); |
||||
|
|
||||
|
// 获取时间的小时 |
||||
|
int six =LocalTime.parse("06:30").getHour(); |
||||
|
|
||||
|
// 时间先后 |
||||
|
boolean isbefore =LocalTime.parse("06:30").isBefore(LocalTime.parse("07:30")); |
||||
|
|
||||
|
// 最大时间 |
||||
|
LocalTime maxTime =LocalTime.MAX; |
||||
|
---- |
||||
|
|
||||
|
==== LocalDateTime |
||||
|
[source,java] |
||||
|
---- |
||||
|
// 当前日期时间 |
||||
|
LocalDateTime now =LocalDateTime.now(); |
||||
|
|
||||
|
// 构建日期时间 |
||||
|
LocalDateTime datetime =LocalDateTime.of(2019, Month.FEBRUARY, 20, 06, 30); |
||||
|
LocalDateTime datetime =LocalDateTime.parse("2019-02-20T06:30:00"); |
||||
|
|
||||
|
// 获取组成部分 |
||||
|
int month =LocalDateTime.now().getMonth(); |
||||
|
|
||||
|
// 加减 |
||||
|
LocalDateTime.now().plusDays(1); |
||||
|
LocalDateTime.now().minusHours(2); |
||||
|
---- |
||||
|
|
||||
|
=== 使用 ZonedDateTime |
||||
|
[source,java] |
||||
|
---- |
||||
|
// 获取上海时区 |
||||
|
ZoneId zoneId =ZoneId.of("Aisa/Shanghai"); |
||||
|
|
||||
|
// 获取所有时区 |
||||
|
Set<String> allZoneIds =ZoneId.getAvailableZoneIds(); |
||||
|
|
||||
|
// 构建时区日期时间 |
||||
|
ZonedDateTime zonedDateTime =ZonedDateTime.of(localDateTime, zoneId); |
||||
|
ZonedDateTime zonedDateTime =ZonedDateTime.parse("2019-06-03T10:15:30+01:00[Aisa/Shanghai]"); |
||||
|
|
||||
|
// 偏移量日期时间 |
||||
|
ZoneOffset offset =ZoneOffset.of("+02:00"); |
||||
|
OffsetDateTime offSetByTwo =OffsetDateTime.of(LocalDateTime.now(), offset); |
||||
|
---- |
||||
|
|
||||
|
=== 使用 Period 和 Duration |
||||
|
* Period : 用于计算两个日期（年月日）间隔。 |
||||
|
* Duration : 用于计算两个时间（秒，纳秒）间隔。 |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
// Period |
||||
|
LocalDate initialDate = LocalDate.parse("2007-05-10"); |
||||
|
LocalDate finalDate = initialDate.plus(Period.ofDays(5)); |
||||
|
|
||||
|
int five = Period.between(finalDate, initialDate).getDays(); |
||||
|
int five = ChronoUnit.DAYS.between(finalDate , initialDate); |
||||
|
|
||||
|
// Duration |
||||
|
LocalTime initialTime = LocalTime.of(6, 30, 0); |
||||
|
LocalTime finalTime = initialTime.plus(Duration.ofSeconds(30)); |
||||
|
|
||||
|
int thirty = Duration.between(finalTime, initialTime).getSeconds(); |
||||
|
int thirty = ChronoUnit.SECONDS.between(finalTime, initialTime); |
||||
|
---- |
||||
|
|
||||
|
|
||||
|
=== Date 和 Calendar 实例转换为新的 Date Time API |
||||
|
[source,java] |
||||
|
---- |
||||
|
LocalDateTime.ofInstant(new Date().toInstant(), ZoneId.systemDefault()); |
||||
|
LocalDateTime.ofInstant(Calendar.getInstance().toInstant(), ZoneId.systemDefault()); |
||||
|
LocalDateTime.ofEpochSecond(1465817690, 0, ZoneOffset.UTC); |
||||
|
---- |
||||
|
|
||||
|
=== 日期和时间格式化 |
||||
|
[source,java] |
||||
|
---- |
||||
|
LocalDateTime localDateTime = LocalDateTime.of(2019, Month.JANUARY, 25, 6, 30); |
||||
|
String localDateString = localDateTime.format(DateTimeFormatter.ISO_DATE); |
||||
|
localDateTime.format(DateTimeFormatter.ofPattern("yyyy/MM/dd")); |
||||
|
localDateTime.format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.MEDIUM).withLocale(Locale.UK); |
||||
|
---- |
||||
|
|
||||
|
== 安全随机数 |
||||
|
[source,java] |
||||
|
---- |
||||
|
SecureRandom random =new SecureRandom(); |
||||
|
random.nextBoolean(); |
||||
|
random.nextBytes(); |
||||
|
random.nextInt(); |
||||
|
random.nextDouble(); |
||||
|
---- |
||||
|
|
||||
|
== 常用的 Http 认证方式 |
||||
|
=== HTTP Basic Authentication |
||||
|
HTTP Basic Authentication 又叫基础认证，它简单地使用 Base64 算法对用户名、密码进行加密，并将加密后的信息放在请求头 Header 中，本质上还是明文传输用户名、密码，并不安全，所以最好在 Https 环境下使用。其认证流程如下： |
||||
|
|
||||
|
image::9999-appendix/java/001.webp[] |
||||
|
|
||||
|
客户端发起 GET 请求 服务端响应返回 401 Unauthorized， www-Authenticate 指定认证算法，realm 指定安全域。然后客户端一般会弹窗提示输入用户名称和密码，输入用户名密码后放入 Header 再次请求，服务端认证成功后以 200 状态码响应客户端。 |
||||
|
|
||||
|
=== HTTP Digest Authentication |
||||
|
为弥补 BASIC 认证存在的弱点就有了 HTTP Digest Authentication 。它又叫摘要认证。它使用随机数加上 MD5 算法来对用户名、密码进行摘要编码，流程类似 Http Basic Authentication ，但是更加复杂一些: |
||||
|
|
||||
|
image::9999-appendix/java/002.webp[] |
||||
|
|
||||
|
. 步骤1：跟基础认证一样，只不过返回带 WWW-Authenticate 首部字段的响应。该字段内包含质问响应方式认证所需要的临时咨询码（随机数，nonce）。 首部字段 WWW-Authenticate 内必须包含 realm 和 nonce 这两个字段的信息。客户端就是依靠向服务器回送这两个值进行认证的。nonce 是一种每次随返回的 401 响应生成的任意随机字符串。该字符串通常推荐由 Base64 编码的十六进制数的组成形式，但实际内容依赖服务器的具体实现 |
||||
|
. 步骤2：接收到 401 状态码的客户端，返回的响应中包含 DIGEST 认证必须的首部字段 Authorization 信息。首部字段 Authorization 内必须包含 username、realm、nonce、uri 和 response 的字段信息，其中，realm 和 nonce 就是之前从服务器接收到的响应中的字段。 |
||||
|
. 步骤3：接收到包含首部字段 Authorization 请求的服务器，会确认认证信息的正确性。认证通过后则会返回包含 Request-URI 资源的响应。 |
||||
|
|
||||
|
并且这时会在首部字段 Authorization-Info 写入一些认证成功的相关信息。 |
||||
|
|
||||
|
=== SSL 客户端认证 |
||||
|
SSL 客户端认证就是通常我们说的 HTTPS 。安全级别较高，但需要承担 CA 证书费用。SSL 认证过程中涉及到一些重要的概念，数字证书机构的公钥、证书的私钥和公钥、非对称算法（配合证书的私钥和公钥使用）、对称密钥、对称算法（配合对称密钥使用）。相对复杂一些这里不过多讲述。 |
||||
|
|
||||
|
=== Form 表单认证 |
||||
|
Form 表单的认证方式并不是HTTP规范。所以实现方式也呈现多样化，其实我们平常的扫码登录，手机验证码登录都属于表单登录的范畴。 |
||||
|
表单认证一般都会配合 Cookie，Session 的使用，现在很多 Web 站点都使用此认证方式。用户在登录页中填写用户名和密码， |
||||
|
服务端认证通过后会将 sessionId 返回给浏览器端，浏览器会保存 sessionId 到浏览器的 Cookie 中。 |
||||
|
因为 HTTP 是无状态的，所以浏览器使用 Cookie 来保存 sessionId。下次客户端会在发送的请求中会携带 sessionId 值， |
||||
|
服务端发现 sessionId 存在并以此为索引获取用户存在服务端的认证信息进行认证操作。认证过则会提供资源访问。 |
||||
|
|
||||
|
登录后返回 JWT Token 一文其实也是通过 Form 提交来获取 Jwt 其实 Jwt 跟 sessionId 同样的作用， |
||||
|
只不过 Jwt 天然携带了用户的一些信息，而 sessionId 需要去进一步获取用户信息。 |
||||
|
|
||||
|
=== Json Web Token 的认证方式 Bearer Authentication |
||||
|
我们通过表单认证获取 Json Web Token ，那么如何使用它呢? |
||||
|
通常我们会把 Jwt 作为令牌使用 Bearer Authentication 方式使用。 |
||||
|
Bearer Authentication 是一种基于令牌的 HTTP 身份验证方案，用户向服务器请求访问受限资源时， |
||||
|
会携带一个 Token 作为凭证，检验通过则可以访问特定的资源。 |
||||
|
最初是在 RFC 6750 中作为 OAuth 2.0 的一部分，但有时也可以单独使用。 |
||||
|
|
||||
|
我们在使用 Bear Token 的方法是在请求头的 Authorization 字段中放入 Bearer <token> 的格式的加密串(Json Web Token)。 |
||||
|
请注意 Bearer 前缀与 Token 之间有一个空字符位，与基本身份验证类似，Bearer Authentication 只能在HTTPS（SSL）上使用。 |
||||
|
|
||||
|
== Java 8 Stream Api |
||||
|
Java 8 提供了非常好用的 Stream API ,可以很方便的操作集合。 |
||||
|
今天我们来探讨两个 Stream 中间操作 map(Function<? super T, ? extends R> mapper) |
||||
|
和 flatMap(Function<? super T, ? extends Stream<? extends R>> mapper) |
||||
|
|
||||
|
=== map 操作 |
||||
|
map 操作是将流中的元素进行再次加工形成一个新流。这在开发中很有用。 |
||||
|
比如我们有一个学生集合，我们需要从中提取学生的年龄以分析学生的年龄分布曲线。 |
||||
|
放在 Java 8 之前 我们要通过新建一个集合然后通过遍历学生集合来消费元素中的年龄属性。 |
||||
|
现在我们通过很简单的流式操作就完成了这个需求。 |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
public class Test { |
||||
|
public static void main(String[] args) { |
||||
|
List<Student> students =new ArrayList<>(); |
||||
|
students.add(new Student("name1",10)); |
||||
|
students.add(new Student("name2",20)); |
||||
|
students.add(new Student("name3",30)); |
||||
|
students.add(new Student("name4",40)); |
||||
|
|
||||
|
List<Integer> ints =students.stream().map(Student::getAge).collect(Collectors.toList()); |
||||
|
for(int i : ints){ |
||||
|
System.out.println(i); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private static class Student { |
||||
|
private String name; |
||||
|
private int age; |
||||
|
|
||||
|
public Student(String name,int age){ |
||||
|
this.name =name; |
||||
|
this.age =age; |
||||
|
} |
||||
|
|
||||
|
public String getName() { |
||||
|
return name; |
||||
|
} |
||||
|
|
||||
|
public void setName(String name) { |
||||
|
this.name = name; |
||||
|
} |
||||
|
|
||||
|
public int getAge() { |
||||
|
return age; |
||||
|
} |
||||
|
|
||||
|
public void setAge(int age) { |
||||
|
this.age = age; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
---- |
||||
|
|
||||
|
=== map 操作 |
||||
|
通过上面的例子，map 操作应该非常好理解。那么 flatMap 是干嘛的呢？ |
||||
|
这样我们把上面的例子给改一下，如果是以班级为单位，提取所有班级下的所有学生的年龄以分析学生的年龄分布曲线。 |
||||
|
这时我们使用上面的方法还行得通吗？ |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
List<List<Student>> studentGroup= gradeList.stream().map(Grade::getStudents).collect(Collectors.toList()); |
||||
|
---- |
||||
|
|
||||
|
通过上面的一顿操作，我们只能得到每个班的学生集合的集合 List<List<Student>>。 |
||||
|
我们还需要嵌套循环才能获取学生的年龄数据，十分不便。如果我们能返回全部学生的集合 List<Students> 就方便多了。 |
||||
|
没错！flatMap 可以搞定！ |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
List<Integer> ages = grades.stream().flatMap(grade -> grade.getStudents().stream()) |
||||
|
.map(Student::getAge).collect(Collectors.toList()); |
||||
|
---- |
||||
|
|
||||
|
正如上面的伪代码所示，我们使用 flatMap 将所有的学生汇聚到一起。 |
||||
|
然后再使用 map 操作提取年龄。 flatMap 不同于 map 地方在于 map 只是提取属性放入流中， |
||||
|
而 flatMap 先提取属性放入一个比较小的流，然后再将所有的流合并为一个流。有一种 “聚沙成塔” 的感觉。 |
||||
|
|
||||
|
=== skip |
||||
|
skip(lang n) 是一个跳过前 n 个元素的中间流操作。我们编写一个简单的方法来进行skip操作,将流剩下的元素打印出来。 |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
public static void skip(long n) { |
||||
|
Stream<Integer> integerStream = Stream.of(10, 9, 8, 7, 6, 5); |
||||
|
integerStream.skip(2).forEach(integer -> System.out.println("integer = " + integer)); |
||||
|
|
||||
|
// out put: |
||||
|
// integer = 8 |
||||
|
// integer = 7 |
||||
|
// integer = 6 |
||||
|
// integer = 5 |
||||
|
} |
||||
|
---- |
||||
|
|
||||
|
skip(long n) 方法跳过前 n （非负）个元素，返回剩下的流，有可能为空流。 |
||||
|
|
||||
|
=== limit |
||||
|
[source,java] |
||||
|
---- |
||||
|
public static void skip(long n) { |
||||
|
Stream<Integer> integerStream = Stream.of(10, 9, 8, 7, 6, 5); |
||||
|
integerStream.limit(2).forEach(integer -> System.out.println("integer = " + integer)); |
||||
|
|
||||
|
// out put: |
||||
|
// integer = 10 |
||||
|
// integer = 9 |
||||
|
} |
||||
|
---- |
||||
|
|
||||
|
=== peek |
||||
|
peek 操作接收的是一个 Consumer<T> 函数。 |
||||
|
顾名思义 peek 操作会按照 Consumer<T> 函数提供的逻辑去消费流中的每一个元素， |
||||
|
同时有可能改变元素内部的一些属性。 |
||||
|
这里我们要提一下这个 Consumer<T> 以理解 什么是消费。 |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
Stream<String> stream = Stream.of("hello", "world"); |
||||
|
List<String> strs= stream.peek(System.out::println).collect(Collectors.toList()); |
||||
|
|
||||
|
// out put: |
||||
|
// hello |
||||
|
// world |
||||
|
---- |
||||
|
peek 操作一般用于不想改变流中元素本身的类型或者只想元素的内部状态时； |
||||
|
而 map 则用于改变流中元素本身类型，即从元素中派生出另一种类型的操作。 |
||||
|
这是他们之间的最大区别。 |
||||
|
|
||||
|
那么 peek 实际中我们会用于哪些场景呢？ |
||||
|
比如对 Stream<T> 中的 T 的某些属性进行批处理的时候用 peek 操作就比较合适。 |
||||
|
如果我们要从 Stream<T> 中获取 T 的某个属性的集合时用 map 也就最好不过了。 |
||||
|
|
||||
|
=== concat |
||||
|
Stream 流合并的前提是元素的类型能够一致 |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
Stream<Integer> stream = Stream.of(1, 2, 3); |
||||
|
Stream<Integer> another = Stream.of(4, 5, 6); |
||||
|
Stream<Integer> concat = Stream.concat(stream, another); |
||||
|
|
||||
|
List<Integer> collect = concat.collect(Collectors.toList()); |
||||
|
List<Integer> expected = Lists.list(1, 2, 3, 4, 5, 6); |
||||
|
|
||||
|
Assertions.assertIterableEquals(expected, collect); |
||||
|
|
||||
|
// 多个流的合并我们也可以使用上面的方式进行“套娃操作”： |
||||
|
Stream.concat(Stream.concat(stream, another), more); |
||||
|
|
||||
|
Stream<Integer> stream = Stream.of(1, 2, 3); |
||||
|
Stream<Integer> another = Stream.of(4, 5, 6); |
||||
|
Stream<Integer> third = Stream.of(7, 8, 9); |
||||
|
Stream<Integer> more = Stream.of(0); |
||||
|
Stream<Integer> concat = Stream.of(stream,another,third,more).flatMap(integerStream -> integerStream); |
||||
|
List<Integer> collect = concat.collect(Collectors.toList()); |
||||
|
List<Integer> expected = Lists.list(1, 2, 3, 4, 5, 6, 7, 8, 9, 0); |
||||
|
Assertions.assertIterableEquals(expected, collect); |
||||
|
---- |
||||
|
|
||||
|
== JOSE |
||||
|
JSON Web Token (JWT) 其实目前已经广为软件开发者所熟知了，但是 JOSE (Javascript Object Signing and Encryption) 却鲜有人知道。 |
||||
|
|
||||
|
=== JOSE 概述 |
||||
|
JOSE 是一种旨在提供在各方之间安全传递声明（claims）的方法的规范集。 |
||||
|
我们常用的 JWT 就包含了允许客户端访问特定应用下特定资源的声明。 |
||||
|
JOSE 制定了一系列的规范来达到此目的。目前该规范还在不断的发展，我们常用的包含以下几个 RFC : |
||||
|
|
||||
|
* JWS（RFC 7515） -JSON Web签名，描述生成和处理签名消息 |
||||
|
* JWE（RFC 7516） -JSON Web加密，描述了保护和处理加密 消息 |
||||
|
* JWK（RFC 7517） -JSON Web密钥，描述 Javascript 对象签名和加密中加密密钥的 格式和处理 |
||||
|
* JWA（RFC 7518） -JSON Web算法，描述了 Javascript 对象签名和加密中使用的 加密 算法 |
||||
|
* JWT（RFC 7519） -JSON Web令牌，描述以 JSON 编码并由 JWS 或 JWE 保护的声明的表示形式 |
||||
|
|
||||
|
=== JWT |
||||
|
JSON Web Token (JWT) is a compact URL-safe means of representing claims to be transferred between two parties. |
||||
|
|
||||
|
直译过来：JSON Web令牌（JWT）是一种紧凑的URL安全方法，用于表示要在两方之间转移的声明。 |
||||
|
|
||||
|
也就是说我们通常说的 JWT 实际上是一个对声明进行 JOSE 处理方式的统称。我们之前用的应该叫 JWS(JSON Web Signature)**， |
||||
|
是 **JWT 的一种实现，除了 JWS , JWT 还有另一种实现 JWE(JSON Web Encryption) 。它们之间的关系应该是这样的： |
||||
|
|
||||
|
image::9999-appendix/java/003.png[] |
||||
|
|
||||
|
=== JWE |
||||
|
JWS 我们就不说了，就是通常我们所说的 JWT。 |
||||
|
JWS 仅仅是对声明(claims)作了签名，保证了其不被篡改，但是其 payload(中段负载) 信息是暴露的。 |
||||
|
也就是 JWS 仅仅能保证数据的完整性而不能保证数据不被泄露。所以我以前也说过它不适合传递敏感数据。 |
||||
|
JWE 的出现就是为了解决这个问题的。具体的可以看下图： |
||||
|
|
||||
|
image::9999-appendix/java/004.png[] |
||||
|
|
||||
|
从上面可以看出 JWE 的生成非常繁琐，作为 Token 可能比较消耗资源和耗时。用作安全的数据传输途径应该不错。 |
||||
|
|
||||
|
=== Spring Security jose 相关 |
||||
|
这里需要简单提一下 Spring Security 提供了 JOSE 有关的类库 spring-security-oauth2-jose ，你可以使用该类库来使用 JOSE 。 |
||||
|
如果 Java 开发者要在 Spring Security 安全框架中使用 OAuth2.0 ，这个类库也是需要研究一下的。 |
||||
|
|
||||
|
== cron 表达式 |
||||
|
cron 表达式是一个字符串，该字符串由 6 个空格分为 7 个域，每一个域代表一个时间含义。 格式如下： |
||||
|
|
||||
|
[秒] [分] [时] [日] [月] [周] [年] |
||||
|
|
||||
|
通常定义 “年” 的部分可以省略，实际常用的由 前六部分组成 |
||||
|
|
||||
|
=== cron 各部定义 |
||||
|
关于 cron 的各个域的定义如下表格所示： |
||||
|
|
||||
|
|=== |
||||
|
| 域 | 是否必填 | 值以及范围 | 通配符 |
||||
|
| 秒 | 是 | 0-59 | , - * / |
||||
|
| 分 | 是 | 0-59 | , - * / |
||||
|
| 时 | 是 | 0-23 | , - * / |
||||
|
| 日 | 是 | 1-31 | , - * ? / L W |
||||
|
| 月 | 是 | 1-12 或 JAN-DEC | , - * / |
||||
|
| 周 | 是 | 1-7 或 SUN-SAT | , - * ? / L # |
||||
|
| 年 | 否 | 1970-2099 | , - * / |
||||
|
|=== |
||||
|
|
||||
|
上面列表中值范围还是比较好理解的，但是比较令开发者难以理解的就是通配符， |
||||
|
其实 cron 表达式的难点也在于通配符。我们在下一个章节进行说明: |
||||
|
|
||||
|
=== cron 中的通配符 |
||||
|
[cols="1,10"] |
||||
|
|=== |
||||
|
| 通配符 | 说明 |
||||
|
| , | 这里指的是在两个以上的时间点中都执行，如果我们在 “分” 这个域中定义为 8,12,35 ，则表示分别在第8分，第12分 第35分执行该定时任务。 |
||||
|
| - | 这个比较好理解就是指定在某个域的连续范围，如果我们在 “时” 这个域中定义 1-6，则表示在1到6点之间每小时都触发一次，用 , 表示 1,2,3,4,5,6 |
||||
|
| * | 表示所有值，可解读为 “每”。 如果在“日”这个域中设置 *,表示每一天都会触发。 |
||||
|
| ? | 表示不指定值。使用的场景为不需要关心当前设置这个字段的值。例如:要在每月的8号触发一个操作，但不关心是周几，我们可以这么设置 0 0 0 8 * ? |
||||
|
| / | 在某个域上周期性触发，该符号将其所在域中的表达式分为两个部分，其中第一部分是起始值，除了秒以外都会降低一个单位，比如 在 “秒” 上定义 5/10 表示从 第 5 秒开始 每 10 秒执行一次，而在 “分” 上则表示从 第 5 秒开始 每 10 分钟执行一次。 |
||||
|
| L | 表示英文中的LAST 的意思，只能在 “日”和“周”中使用。在“日”中设置，表示当月的最后一天(依据当前月份，如果是二月还会依据是否是润年), 在“周”上表示周六，相当于”7”或”SAT”。如果在”L”前加上数字，则表示该数据的最后一个。例如在“周”上设置”7L”这样的格式,则表示“本月最后一个周六” |
||||
|
| W | 表示离指定日期的最近那个工作日(周一至周五)触发，只能在 “日” 中使用且只能用在具体的数字之后。若在“日”上置”15W”，表示离每月15号最近的那个工作日触发。假如15号正好是周六，则找最近的周五(14号)触发, 如果15号是周未，则找最近的下周一(16号)触发.如果15号正好在工作日(周一至周五)，则就在该天触发。如果是 “1W” 就只能往本月的下一个最近的工作日推不能跨月往上一个月推。 |
||||
|
| # | 表示每月的第几个周几，只能作用于 “周” 上。例如 ”2#3” 表示在每月的第三个周二。 |
||||
|
|=== |
||||
|
|
||||
|
=== 示例 |
||||
|
[source,bash] |
||||
|
---- |
||||
|
每隔1分钟执行一次：0 */1 * * * ? |
||||
|
每天22点执行一次：0 0 22 * * ? |
||||
|
每月1号凌晨1点执行一次：0 0 1 1 * ? |
||||
|
每月最后一天23点执行一次：0 0 23 L * ? |
||||
|
每周周六凌晨3点实行一次：0 0 3 ? * L |
||||
|
在24分、30分执行一次：0 24,30 * * * ? |
||||
|
---- |
||||
|
|
||||
|
== java 8 Collectors |
||||
|
Collectors 是 Java 8 加入的操作类，位于 java.util.stream 包下。 |
||||
|
它会根据不同的策略将元素收集归纳起来，比如最简单常用的是将元素装入 Map、Set、List 等可变容器中。 |
||||
|
特别对于 Java 8 Stream Api 来说非常有用。它提供了collect() 方法来对 Stream 流进行终结操作派生出基于各种策略的结果集。 |
||||
|
我们就借助于 Stream 来熟悉一下 Collectors 吧。 |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
public class Student { |
||||
|
private String id; |
||||
|
private String name; |
||||
|
private LocalDate birthday; |
||||
|
private int age; |
||||
|
private double score; |
||||
|
|
||||
|
public Student(String id,String name,LocalDate birthday,int age,double score){ |
||||
|
this.id =id; |
||||
|
this.name =name; |
||||
|
this.birthday =birthday; |
||||
|
this.age =age; |
||||
|
this.score =score; |
||||
|
} |
||||
|
|
||||
|
public String getId() { |
||||
|
return id; |
||||
|
} |
||||
|
|
||||
|
public void setId(String id) { |
||||
|
this.id = id; |
||||
|
} |
||||
|
|
||||
|
public String getName() { |
||||
|
return name; |
||||
|
} |
||||
|
|
||||
|
public void setName(String name) { |
||||
|
this.name = name; |
||||
|
} |
||||
|
|
||||
|
public LocalDate getBirthday() { |
||||
|
return birthday; |
||||
|
} |
||||
|
|
||||
|
public void setBirthday(LocalDate birthday) { |
||||
|
this.birthday = birthday; |
||||
|
} |
||||
|
|
||||
|
public int getAge() { |
||||
|
return age; |
||||
|
} |
||||
|
|
||||
|
public void setAge(int age) { |
||||
|
this.age = age; |
||||
|
} |
||||
|
|
||||
|
public double getScore() { |
||||
|
return score; |
||||
|
} |
||||
|
|
||||
|
public void setScore(double score) { |
||||
|
this.score = score; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public String toString() { |
||||
|
return "(" + |
||||
|
"id='" + id + '\'' + |
||||
|
", name='" + name + '\'' + |
||||
|
", birthday=" + birthday + |
||||
|
", age=" + age + |
||||
|
", score=" + score + |
||||
|
')'; |
||||
|
} |
||||
|
|
||||
|
public static void main(String[] args) { |
||||
|
final List<Student> students =new ArrayList<>(); |
||||
|
students.add(new Student("1", "张三", LocalDate.of(2009, Month.JANUARY, 1), 12, 12.123)); |
||||
|
students.add(new Student("2", "李四", LocalDate.of(2010, Month.FEBRUARY, 2), 11, 22.123)); |
||||
|
students.add(new Student("3", "王五", LocalDate.of(2011, Month.MARCH, 3), 10, 32.123)); |
||||
|
students.add(new Student("4", "赵六", LocalDate.of(2011, Month.FEBRUARY, 3), 10, 50.123)); |
||||
|
|
||||
|
// counting |
||||
|
long count =students.stream().collect(Collectors.counting()); |
||||
|
System.out.println("counting: " + count); |
||||
|
// counting: 4 |
||||
|
|
||||
|
// averagingDouble |
||||
|
double averagingDouble =students.stream().collect(Collectors.averagingDouble(Student::getScore)); |
||||
|
System.out.println("averagingDouble: " + averagingDouble); |
||||
|
// averagingDouble: 29.122999999999998 |
||||
|
|
||||
|
// averagingInt |
||||
|
double averagingInt =students.stream().collect(Collectors.averagingInt(Student::getAge)); |
||||
|
System.out.println("averagingInt: " + averagingInt); |
||||
|
// averagingInt: 10.75 |
||||
|
|
||||
|
// averagingLong |
||||
|
double averagingLong =students.stream().collect(Collectors.averagingLong(Student::getAge)); |
||||
|
System.out.println("averagingLong: " + averagingLong); |
||||
|
// averagingLong: 10.75 |
||||
|
|
||||
|
// summingInt |
||||
|
int summingInt =students.stream().collect(Collectors.summingInt(Student::getAge)); |
||||
|
System.out.println("summingInt: " + summingInt); |
||||
|
// summingInt: 43 |
||||
|
|
||||
|
// summingLong |
||||
|
long summingLong =students.stream().collect(Collectors.summingLong(Student::getAge)); |
||||
|
System.out.println("summingLong: " + summingLong); |
||||
|
// summingLong: 43 |
||||
|
|
||||
|
// summingDouble |
||||
|
double summingDouble =students.stream().collect(Collectors.summingDouble(Student::getAge)); |
||||
|
System.out.println("summingDouble: " + summingDouble); |
||||
|
// summingDouble: 43.0 |
||||
|
|
||||
|
// minBy |
||||
|
Optional<Student> minBy =students.stream().collect(Collectors.minBy(Comparator.comparing(Student::getAge))); |
||||
|
System.out.println("minBy: " + minBy.get()); |
||||
|
// minBy: (id='3', name='王五', birthday=2011-03-03, age=10, score=32.123) |
||||
|
|
||||
|
// maxBy |
||||
|
Optional<Student> maxBy =students.stream().collect(Collectors.maxBy(Comparator.comparing(Student::getAge))); |
||||
|
System.out.println("maxBy: " + maxBy.get()); |
||||
|
// maxBy: (id='1', name='张三', birthday=2009-01-01, age=12, score=12.123) |
||||
|
|
||||
|
// toList |
||||
|
final List<String> idList = students.stream().map(Student::getId).collect(Collectors.toList()); |
||||
|
System.out.println("toList: " + idList); |
||||
|
// toList: [1, 2, 3, 4] |
||||
|
|
||||
|
// toSet |
||||
|
final Set<String> idSet = students.stream().map(Student::getId).collect(Collectors.toSet()); |
||||
|
System.out.println("toSet: " + idSet); |
||||
|
// toSet: [1, 2, 3, 4] |
||||
|
|
||||
|
// toCollection |
||||
|
final Collection<String> idTreeSet = students.stream().map(Student::getId).collect(Collectors.toCollection(TreeSet::new)); |
||||
|
System.out.println("toCollection: " + idTreeSet); |
||||
|
// toCollection: [1, 2, 3, 4] |
||||
|
|
||||
|
// toMap1 |
||||
|
final Map<String, Student> map1 = students.stream().collect(Collectors.toMap(Student::getId, Function.identity())); |
||||
|
System.out.println("toMap1: " + map1); |
||||
|
// toMap1: {1=(id='1', name='张三', birthday=2009-01-01, age=12, score=12.123), 2=(id='2', name='李四', birthday=2010-02-02, age=11, score=22.123), 3=(id='3', name='王五', birthday=2011-03-03, age=10, score=32.123), 4=(id='4', name='赵六', birthday=2011-02-03, age=10, score=50.123)} |
||||
|
|
||||
|
|
||||
|
// toMap2 |
||||
|
final Map<String,Student> map2 =students.stream().collect(Collectors.toMap(Student::getId,Function.identity(),(x,y)->x)); |
||||
|
System.out.println("toMap2: " + map2); |
||||
|
// toMap2: {1=(id='1', name='张三', birthday=2009-01-01, age=12, score=12.123), 2=(id='2', name='李四', birthday=2010-02-02, age=11, score=22.123), 3=(id='3', name='王五', birthday=2011-03-03, age=10, score=32.123), 4=(id='4', name='赵六', birthday=2011-02-03, age=10, score=50.123)} |
||||
|
|
||||
|
// toMap3 |
||||
|
final Map<String, String> map3 = students.stream().collect(Collectors.toMap(Student::getId, Student::getName, (x, y) -> x)); |
||||
|
System.out.println("toMap3: " + map3); |
||||
|
// toMap3: {1=张三, 2=李四, 3=王五, 4=赵六} |
||||
|
|
||||
|
// toMap4 年龄相同的采用分数最高的 |
||||
|
final Map<Integer, Student> map4 = students.stream().collect(Collectors.toMap(Student::getAge, Function.identity(), BinaryOperator.maxBy(Comparator.comparing(Student::getScore)))); |
||||
|
System.out.println("toMap4: " + map4); |
||||
|
// toMap4: {10=(id='4', name='赵六', birthday=2011-02-03, age=10, score=50.123), 11=(id='2', name='李四', birthday=2010-02-02, age=11, score=22.123), 12=(id='1', name='张三', birthday=2009-01-01, age=12, score=12.123)} |
||||
|
|
||||
|
// groupingBy |
||||
|
final Map<Integer, List<Student>> group1 = students.stream().collect(Collectors.groupingBy(Student::getAge)); |
||||
|
System.out.println("groupingBy: " + group1); |
||||
|
// groupingBy: {10=[(id='3', name='王五', birthday=2011-03-03, age=10, score=32.123), (id='4', name='赵六', birthday=2011-02-03, age=10, score=50.123)], 11=[(id='2', name='李四', birthday=2010-02-02, age=11, score=22.123)], 12=[(id='1', name='张三', birthday=2009-01-01, age=12, score=12.123)]} |
||||
|
|
||||
|
// groupingBy |
||||
|
final Map<Integer, Set<Student>> group2 = students.stream().collect(Collectors.groupingBy(Student::getAge, Collectors.toSet())); |
||||
|
System.out.println("groupingBy2: " + group2); |
||||
|
// groupingBy2: {10=[(id='3', name='王五', birthday=2011-03-03, age=10, score=32.123), (id='4', name='赵六', birthday=2011-02-03, age=10, score=50.123)], 11=[(id='2', name='李四', birthday=2010-02-02, age=11, score=22.123)], 12=[(id='1', name='张三', birthday=2009-01-01, age=12, score=12.123)]} |
||||
|
|
||||
|
// partitioningBy |
||||
|
// partitioningBy 与 groupingBy 的区别在于，partitioningBy 借助 Predicate 断言，可以将集合元素分为 true 和 false 两部分。比如，按照年龄是否大于 11 分组： |
||||
|
final Map<Boolean, List<Student>> partitioningBy = students.stream().collect(Collectors.partitioningBy(s -> s.getAge() > 11)); |
||||
|
System.out.println("partitioningBy: " + partitioningBy); |
||||
|
// partitioningBy: {false=[(id='2', name='李四', birthday=2010-02-02, age=11, score=22.123), (id='3', name='王五', birthday=2011-03-03, age=10, score=32.123), (id='4', name='赵六', birthday=2011-02-03, age=10, score=50.123)], true=[(id='1', name='张三', birthday=2009-01-01, age=12, score=12.123)]} |
||||
|
|
||||
|
// partitioningBy |
||||
|
final Map<Boolean, Set<Student>> partitioningBy2 = students.stream().collect(Collectors.partitioningBy(s -> s.getAge() > 11,Collectors.toSet())); |
||||
|
System.out.println("partitioningBy2: " + partitioningBy2); |
||||
|
// partitioningBy2: {false=[(id='3', name='王五', birthday=2011-03-03, age=10, score=32.123), (id='4', name='赵六', birthday=2011-02-03, age=10, score=50.123), (id='2', name='李四', birthday=2010-02-02, age=11, score=22.123)], true=[(id='1', name='张三', birthday=2009-01-01, age=12, score=12.123)]} |
||||
|
|
||||
|
// joining |
||||
|
System.out.println("joining: " + Stream.of("java", "go", "sql").collect(Collectors.joining())); |
||||
|
// joining: javagosql |
||||
|
|
||||
|
// joining2 |
||||
|
System.out.println("joining2: " + Stream.of("java", "go", "sql").collect(Collectors.joining(","))); |
||||
|
// joining2: java,go,sql |
||||
|
|
||||
|
// joining3 |
||||
|
System.out.println("joining3: " + Stream.of("java", "go", "sql").collect(Collectors.joining(",","(",")"))); |
||||
|
// joining3: (java,go,sql) |
||||
|
|
||||
|
// mapping |
||||
|
System.out.println("mapping: " + students.stream().collect(Collectors.mapping(Student::getName, Collectors.toList()))); |
||||
|
// mapping: [张三, 李四, 王五, 赵六] |
||||
|
|
||||
|
System.out.println("mapping2: " + students.stream().map(Student::getName).collect(Collectors.toList())); |
||||
|
// mapping2: [张三, 李四, 王五, 赵六] |
||||
|
|
||||
|
// reducing |
||||
|
System.out.println("reducing: " + students.stream().map(Student::getScore).collect(Collectors.reducing(Double::sum))); |
||||
|
// reducing: Optional[116.49199999999999] |
||||
|
|
||||
|
// reducing2 |
||||
|
System.out.println("reducing2: " + students.stream().map(Student::getScore).collect(Collectors.reducing(10.0,Double::sum))); |
||||
|
// reducing2: 126.49199999999999 |
||||
|
|
||||
|
// reducing3 |
||||
|
System.out.println("reducing3: " + students.stream().collect(Collectors.reducing(0.0, Student::getScore, Double::sum))); |
||||
|
// reducing3: 116.49199999999999 |
||||
|
|
||||
|
// reducing4 |
||||
|
System.out.println("reducing4: " + students.stream().map(Student::getScore).reduce(0.0,Double::sum)); |
||||
|
// reducing4: 116.49199999999999 |
||||
|
} |
||||
|
} |
||||
|
---- |
||||
|
|
||||
|
== Java Collection 移除元素 |
||||
|
操作集合是一个 Java 编程人员几乎每天都在重复的事情。 |
||||
|
|
||||
|
=== for 循环并不一定能从集合中移除元素 |
||||
|
[source,java] |
||||
|
---- |
||||
|
for (String server : servers) { |
||||
|
if (server.startsWith("F")) { |
||||
|
servers.remove(server); |
||||
|
} |
||||
|
} |
||||
|
---- |
||||
|
使用传统的 foreach 循环移除 F 开头的假服务器，但是你会发现这种操作引发了 ConcurrentModificationException 异常。 |
||||
|
|
||||
|
难道 for 循环就不能移除元素了吗？当然不是！我们如果能确定需要被移除的元素的索引还是可以的。 |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
for (int i = 0; i < servers.size(); i++) { |
||||
|
if (servers.get(i).startsWith("F")) { |
||||
|
servers.remove(i); |
||||
|
} |
||||
|
} |
||||
|
---- |
||||
|
|
||||
|
但是这种方式我目前只演示了 ArrayList，其它的类型并没有严格测试，留给你自己探索。 |
||||
|
|
||||
|
=== 迭代器 Iterator 可以删除集合中的元素 |
||||
|
在传统方式中我们使用 Iterator 是可以保证删除元素的： |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
Iterator<String> iterator = servers.iterator(); |
||||
|
while (iterator.hasNext()) { |
||||
|
String next = iterator.next(); |
||||
|
if (next.startsWith("F")) { |
||||
|
iterator.remove(); |
||||
|
} |
||||
|
} |
||||
|
---- |
||||
|
|
||||
|
=== 遍历删除元素的缺点 |
||||
|
* 我们需要遍历集合的每一个元素并对它们进行断言，哪怕你删除一个元素。 |
||||
|
* 尽管我们可以通过迭代的方式删除特定的元素，但是操作繁琐，根据集合类型的不同有潜在的 ConcurrentModificationException 异常。 |
||||
|
* 根据数据结构的不同，删除元素的时间复杂度也大大不同。比如数组结构的 ArrayList 在删除元素的速度上不如链表结构的 LinkedList。 |
||||
|
|
||||
|
=== 新的集合元素删除操作 |
||||
|
Java 8 提供了新的集合操作 API 和 Stream 来帮助我们解决这个问题。 |
||||
|
|
||||
|
==== Collection.removeIf() |
||||
|
新的 Collection Api removeIf(Predicate<? super E> filter) 。 |
||||
|
该 Api 提供了一种更简洁的使用 Predicate （断言）删除元素的方法，于是我们可以更加简洁的实现开始的需求： |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
servers.removeIf(s-> s.startsWith("F")); |
||||
|
---- |
||||
|
|
||||
|
=== 通过 filter 断言实现 |
||||
|
我们可以使用 Stream 的 filter 断言。filter 断言会把符合断言的流元素汇集成一个新的流，然后归纳起来即可，于是我们可以这么写： |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
List<String> newServers = servers.stream().filter(s -> !s.startsWith("F")).collect(Collectors.toList()); |
||||
|
---- |
||||
|
|
||||
|
这个优点上面已经说了不会影响原始数据，生成的是一个副本。缺点就是可能会有内存占用问题。 |
||||
|
|
||||
|
|
||||
|
|
||||
|
|
||||
|
|
||||
@ -0,0 +1,70 @@ |
|||||
|
[appendix] |
||||
|
= OAuth2 |
||||
|
OAuth2 是一个开放标准,允许用户授权第三方应用程序访问他们存储在另外的服务提供者上的信息，不需要将用户名和密码提供给第三方应用。 |
||||
|
|
||||
|
OAuth2 协议允许用户将认证与授权交给一个独立的第三方进行担保。 |
||||
|
|
||||
|
OAuth2可以提供一个统一的认证服务。 |
||||
|
|
||||
|
== 模块构成 |
||||
|
* Resource owner (资源拥有者): 拥有该资源的服务或用户，如我们自己或者资源网站 |
||||
|
* Authorization server (认证服务器) : 即用来认证与颁发令牌（如token）的服务 |
||||
|
* Resource server (资源服务器) : 拥有资源的服务，如我们要访问的网站 |
||||
|
* Client (客户端) : 即访问的客户端，如我们自己用的访问网站 |
||||
|
|
||||
|
== 授权方式 |
||||
|
* authorization_code (授权码模式) : 最正规的模式，客户端先将用户导向认证服务器，登录后获取授权码，然后进行授权，最后根据授权码获取访问令牌 |
||||
|
* refresh_token (刷新模式) : 用刷新码获取 |
||||
|
* client_credentials (客户端模式) : 第三方应用自己本身需要获取资源 |
||||
|
|
||||
|
== 认证方式 |
||||
|
* client_secret_basic : 客户端认证信息会以 Basic Auth 的方式进行传递 |
||||
|
* client_secret_post : 客户端认证信息放在请求体中进行传递 |
||||
|
* client_secret_jwt : HMAC 算法生成 JWT 来传递客户端秘钥 |
||||
|
* private_key_jwt : 客户端使用 RSA 和 EC 算法生成 JWT 传递客户端认证信息，需提供公钥给授权服务器 |
||||
|
* none : 会开启 PKCE 功能以确保安全, 适应于开放型客户端 |
||||
|
|
||||
|
=== client_secret_jwt |
||||
|
OAuth2 客户端将自己的密钥作为 HMAC SHA256 算法的 key 生成 SecretKey |
||||
|
|
||||
|
[source,java] |
||||
|
---- |
||||
|
byte[] pin =clientSecret.getBytes(StandardCharsets.UTF_8); |
||||
|
SecretKeySpec secretKey =new SecretKeySpec(pin,"HmacSHA256"); |
||||
|
---- |
||||
|
|
||||
|
然后通过 SecretKey 生成一个携带 OAuth2 客户端信息的 JWT，在授权码请求 Token 环节携带该 JWT 以便授权服务器进行客户端认证，请求的报文为: |
||||
|
|
||||
|
[source,html] |
||||
|
---- |
||||
|
POST /oauth2/token HTTP/1.1 |
||||
|
Host: oauth2_client.felord.cn |
||||
|
Content-Type: application/x-www-form-urlencoded |
||||
|
|
||||
|
grant_type=authorization_code& |
||||
|
code=n0esc3NRze7LTCu7iYzS6a5acc3f0ogp4& |
||||
|
client_assertion_type=urn:ietf:params:oauth:client-assertion-type:jwt-bearer& |
||||
|
client_assertion=你的JWT |
||||
|
---- |
||||
|
|
||||
|
授权服务器收到请求后通过 OAuth2 客户端的 client_secret 对 JWT 进行解码校验以认证客户端。这种方式能很好的保护 client_secret 在非 HTTPS 环境下的传输。 |
||||
|
这里 OAuth2 客户端的密钥（client_secret）比特长度必须大于等于256。 |
||||
|
|
||||
|
=== private_key_jwt |
||||
|
private_key_jwt 和 client_secret_jwt 唯一的区别就是生成 JWT 的方式不同。 |
||||
|
通过这种方式，OAuth2 客户端已经不需要 client_secret，只需要配置一对 RSA 或者 EC 密钥，通过密钥来生成 JWT， |
||||
|
另外还需要向授权服务器提供公钥，通常是一个 jwkSetUrl。这种方式让客户端的认证信息更加安全的传输。 |
||||
|
|
||||
|
== 请求点 |
||||
|
|=== |
||||
|
| method | endpoint | 说明 |
||||
|
| GET | /.well-known/oauth-authorization-server | meta data |
||||
|
| GET | /.well-known/openid-configuration | meta data |
||||
|
| GET/POST | /oauth2/authorize | 获取授权码 |
||||
|
| POST | /oauth2/token | 获取访问令牌 |
||||
|
| POST | /oauth2/introspect | 内省 |
||||
|
| POST | /oauth2/revoke | 收回令牌 |
||||
|
| GET/POST | /userinfo | 用户信息 |
||||
|
| GET | /oauth2/jwks | JWK(JSON Web 密钥) + JWS(JSON Web 签名) |
||||
|
| POST | /connect/register | oidc 客户端注册 |
||||
|
|=== |
||||
|
After Width: | Height: | Size: 51 KiB |
|
After Width: | Height: | Size: 62 KiB |
|
After Width: | Height: | Size: 23 KiB |
|
After Width: | Height: | Size: 91 KiB |
@ -1,104 +1,102 @@ |
|||||
{ |
{ |
||||
"name": "io.sc.platform.developer.frontend", |
"name": "io.sc.platform.developer.frontend", |
||||
"version": "8.1.20", |
"version": "8.1.20", |
||||
"description": "", |
"description": "", |
||||
"private": false, |
"private": false, |
||||
"keywords": [ |
"keywords": [], |
||||
|
"author": "", |
||||
], |
"license": "ISC", |
||||
"author": "", |
"scripts": { |
||||
"license": "ISC", |
"clean": "rm -rf ./node_modules && rm -rf pnpm-lock.yaml", |
||||
"scripts": { |
"dev": "nodemon", |
||||
"clean": "rm -rf ./node_modules && rm -rf pnpm-lock.yaml", |
"serve": "node ./util-components-generator.cjs && cross-env NODE_ENV=development webpack serve --config webpack.env.serve.cjs", |
||||
"dev": "nodemon", |
"build": "node ./util-components-generator.cjs && cross-env NODE_ENV=development webpack --config webpack.env.build.cjs", |
||||
"serve": "node ./util-components-generator.cjs && cross-env NODE_ENV=development webpack serve --config webpack.env.serve.cjs", |
"prod": "node ./util-components-generator.cjs && cross-env NODE_ENV=production webpack --config webpack.env.prod.cjs", |
||||
"build": "node ./util-components-generator.cjs && cross-env NODE_ENV=development webpack --config webpack.env.build.cjs", |
"sync": "platform sync" |
||||
"prod": "node ./util-components-generator.cjs && cross-env NODE_ENV=production webpack --config webpack.env.prod.cjs", |
}, |
||||
"sync": "platform sync" |
"engines": { |
||||
}, |
"node": ">=18", |
||||
"engines": { |
"pnpm": ">=7" |
||||
"node": ">=18", |
}, |
||||
"pnpm": ">=7" |
"publishConfig": { |
||||
}, |
"registry": "http://nexus.sc.io:8000/repository/npm-releases/", |
||||
"publishConfig": { |
"access": "public" |
||||
"registry": "http://nexus.sc.io:8000/repository/npm-releases/", |
}, |
||||
"access": "public" |
"devDependencies": { |
||||
}, |
"@babel/core": "7.23.7", |
||||
"devDependencies": { |
"@babel/preset-env": "7.23.7", |
||||
"@babel/core": "7.23.2", |
"@babel/preset-typescript": "7.23.3", |
||||
"@babel/preset-env": "7.23.2", |
"@babel/plugin-transform-class-properties": "7.23.3", |
||||
"@babel/preset-typescript": "7.23.2", |
"@babel/plugin-transform-object-rest-spread": "7.23.4", |
||||
"@babel/plugin-transform-class-properties": "7.22.5", |
"@quasar/app-webpack": "3.12.1", |
||||
"@babel/plugin-transform-object-rest-spread": "7.22.15", |
"@quasar/cli": "2.3.0", |
||||
"@quasar/app-webpack": "3.11.2", |
"@types/mockjs": "1.0.10", |
||||
"@quasar/cli": "2.3.0", |
"@types/node": "20.10.6", |
||||
"@types/mockjs": "1.0.9", |
"@typescript-eslint/eslint-plugin": "6.17.0", |
||||
"@types/node": "20.8.9", |
"@typescript-eslint/parser": "6.17.0", |
||||
"@typescript-eslint/eslint-plugin": "6.9.0", |
"@vue/compiler-sfc": "3.4.3", |
||||
"@typescript-eslint/parser": "6.9.0", |
"@webpack-cli/serve": "2.0.5", |
||||
"@vue/compiler-sfc": "3.3.7", |
"autoprefixer": "10.4.16", |
||||
"@webpack-cli/serve": "2.0.5", |
"babel-loader": "9.1.3", |
||||
"autoprefixer": "10.4.16", |
"clean-webpack-plugin": "4.0.0", |
||||
"babel-loader": "9.1.3", |
"copy-webpack-plugin": "11.0.0", |
||||
"clean-webpack-plugin": "4.0.0", |
"cross-env": "7.0.3", |
||||
"copy-webpack-plugin": "11.0.0", |
"css-loader": "6.8.1", |
||||
"cross-env": "7.0.3", |
"eslint": "8.56.0", |
||||
"css-loader": "6.8.1", |
"eslint-config-prettier": "9.1.0", |
||||
"eslint": "8.52.0", |
"eslint-plugin-prettier": "5.1.2", |
||||
"eslint-config-prettier": "9.0.0", |
"eslint-plugin-vue": "9.19.2", |
||||
"eslint-plugin-prettier": "5.0.1", |
"eslint-webpack-plugin": "4.0.1", |
||||
"eslint-plugin-vue": "9.18.0", |
"html-webpack-plugin": "5.6.0", |
||||
"eslint-webpack-plugin": "4.0.1", |
"json5": "2.2.3", |
||||
"html-webpack-plugin": "5.5.3", |
"mini-css-extract-plugin": "2.7.6", |
||||
"json5": "2.2.3", |
"nodemon": "3.0.2", |
||||
"mini-css-extract-plugin": "2.7.6", |
"postcss": "8.4.32", |
||||
"nodemon": "3.0.1", |
"postcss-import": "16.0.0", |
||||
"postcss": "8.4.31", |
"postcss-loader": "7.3.4", |
||||
"postcss-import": "15.1.0", |
"postcss-preset-env": "9.3.0", |
||||
"postcss-loader": "7.3.3", |
"prettier": "3.1.1", |
||||
"postcss-preset-env": "9.2.0", |
"sass": "1.69.7", |
||||
"prettier": "3.0.3", |
"sass-loader": "13.3.3", |
||||
"sass": "1.69.5", |
"typescript": "5.3.3", |
||||
"sass-loader": "13.3.2", |
"vue-loader": "17.4.2", |
||||
"typescript": "5.2.2", |
"webpack": "5.89.0", |
||||
"vue-loader": "17.3.0", |
"webpack-bundle-analyzer": "4.10.1", |
||||
"webpack": "5.89.0", |
"webpack-cli": "5.1.4", |
||||
"webpack-bundle-analyzer": "4.9.1", |
"webpack-dev-server": "4.15.1", |
||||
"webpack-cli": "5.1.4", |
"webpack-merge": "5.10.0" |
||||
"webpack-dev-server": "4.15.1", |
}, |
||||
"webpack-merge": "5.10.0" |
"dependencies": { |
||||
}, |
"@quasar/extras": "1.16.9", |
||||
"dependencies": { |
"@vueuse/core": "10.7.1", |
||||
"@quasar/extras": "1.16.7", |
"axios": "1.6.3", |
||||
"@vueuse/core": "10.3.0", |
"dayjs": "1.11.10", |
||||
"axios": "1.5.1", |
"echarts": "5.4.3", |
||||
"dayjs": "1.11.10", |
"exceljs": "4.4.0", |
||||
"echarts": "5.4.1", |
"file-saver": "2.0.5", |
||||
"exceljs": "4.3.0", |
"luckyexcel": "1.0.1", |
||||
"file-saver": "2.0.5", |
"mockjs": "1.1.0", |
||||
"luckyexcel": "1.0.1", |
"pinia": "2.1.7", |
||||
"mockjs": "1.1.0", |
"platform-core": "8.1.49", |
||||
"pinia": "2.1.7", |
"quasar": "2.14.2", |
||||
"platform-core": "8.1.47", |
"tailwindcss": "3.4.0", |
||||
"quasar": "2.13.0", |
"vue": "3.4.3", |
||||
"tailwindcss": "3.3.5", |
"vue-dompurify-html": "5.0.1", |
||||
"vue": "3.3.7", |
"vue-i18n": "9.8.0", |
||||
"vue-dompurify-html": "4.1.4", |
"vue-router": "4.2.5", |
||||
"vue-i18n": "9.6.0", |
"@codemirror/autocomplete": "6.11.1", |
||||
"vue-router": "4.2.5", |
"@codemirror/commands": "6.3.3", |
||||
"@codemirror/autocomplete": "6.11.1", |
"@codemirror/lang-html": "6.4.7", |
||||
"@codemirror/commands": "6.3.2", |
"@codemirror/lang-java": "6.0.1", |
||||
"@codemirror/lang-html": "6.4.7", |
"@codemirror/lang-javascript": "6.2.1", |
||||
"@codemirror/lang-java": "6.0.1", |
"@codemirror/lang-json": "6.0.1", |
||||
"@codemirror/lang-javascript": "6.2.1", |
"@codemirror/lang-sql": "6.5.5", |
||||
"@codemirror/lang-json": "6.0.1", |
"@codemirror/lang-xml": "6.0.2", |
||||
"@codemirror/lang-sql": "6.5.4", |
"@codemirror/language": "6.10.0", |
||||
"@codemirror/lang-xml": "6.0.2", |
"@codemirror/search": "6.5.5", |
||||
"@codemirror/language": "6.9.3", |
"@codemirror/state": "6.4.0", |
||||
"@codemirror/search": "6.5.5", |
"@codemirror/view": "6.23.0", |
||||
"@codemirror/state": "6.3.3", |
"codemirror": "6.0.1", |
||||
"@codemirror/view": "6.22.1", |
"vue-codemirror6": "1.2.0" |
||||
"codemirror": "6.0.1", |
} |
||||
"vue-codemirror6": "1.1.31" |
|
||||
} |
|
||||
} |
} |
||||
@ -0,0 +1,66 @@ |
|||||
|
package io.sc.platform.installer.item; |
||||
|
|
||||
|
import io.sc.platform.core.DirectoryManager; |
||||
|
import io.sc.platform.core.service.ApplicationPropertiesFileService; |
||||
|
import io.sc.platform.core.support.ProgressInfo; |
||||
|
import io.sc.platform.core.util.FileUtil; |
||||
|
import io.sc.platform.core.util.PropertiesX; |
||||
|
import io.sc.platform.core.util.StringUtil; |
||||
|
import io.sc.platform.core.util.UrlUtil; |
||||
|
import io.sc.platform.installer.InstallerItem; |
||||
|
import org.jasypt.encryption.StringEncryptor; |
||||
|
import org.springframework.context.ApplicationContext; |
||||
|
import org.springframework.util.StringUtils; |
||||
|
|
||||
|
import java.util.Map; |
||||
|
|
||||
|
public class ServerInstallerItem implements InstallerItem { |
||||
|
@Override |
||||
|
public int getOrder() { |
||||
|
return 1500; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public int getWeight() { |
||||
|
return 5; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public String getTemplateLoaction() { |
||||
|
return "io/sc/platform/installer/installer_server.html"; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void install(Map<String, String> config, ProgressInfo progressInfo, ApplicationContext applicationContext) throws Exception { |
||||
|
ApplicationPropertiesFileService applicationPropertiesFileService =applicationContext.getBean(ApplicationPropertiesFileService.class); |
||||
|
PropertiesX propertiesX =applicationPropertiesFileService.getPropertiesX(); |
||||
|
|
||||
|
String protocol =config.get("protocol"); |
||||
|
if("https".equalsIgnoreCase(protocol)){ |
||||
|
propertiesX.setProperty("server.ssl.enabled","true"); |
||||
|
propertiesX.setProperty("server.ssl.key-store","classpath:keystore/keystore.p12"); |
||||
|
propertiesX.setProperty("server.ssl.key-store-password","ENC(BWQMgGHmKLMSOsLQgyf1ejQl45HER/XG)"); |
||||
|
propertiesX.setProperty("server.ssl.keyStoreType","PKCS12"); |
||||
|
propertiesX.setProperty("server.ssl.keyAlias","platform"); |
||||
|
} |
||||
|
|
||||
|
String ip =config.get("ip"); |
||||
|
String port =config.get("port"); |
||||
|
String webContextPath =config.get("webContextPath"); |
||||
|
|
||||
|
if(StringUtils.hasText(ip)) { |
||||
|
propertiesX.setProperty("server.address", ip); |
||||
|
} |
||||
|
|
||||
|
propertiesX.setProperty("server.port",port); |
||||
|
|
||||
|
webContextPath =UrlUtil.removeUrlPrefixSlash(webContextPath); |
||||
|
webContextPath =UrlUtil.removeUrlSuffixSlash(webContextPath); |
||||
|
if(StringUtils.hasText(webContextPath)) { |
||||
|
webContextPath = "/" + webContextPath + "/"; |
||||
|
}else{ |
||||
|
webContextPath = "/"; |
||||
|
} |
||||
|
propertiesX.setProperty("server.servlet.context-path",webContextPath); |
||||
|
} |
||||
|
} |
||||
@ -1,4 +1,5 @@ |
|||||
io.sc.platform.installer.item.WelcomeInstallerItem |
io.sc.platform.installer.item.WelcomeInstallerItem |
||||
io.sc.platform.installer.item.TypeInstallerItem |
io.sc.platform.installer.item.TypeInstallerItem |
||||
|
io.sc.platform.installer.item.ServerInstallerItem |
||||
io.sc.platform.installer.item.SummaryInstallerItem |
io.sc.platform.installer.item.SummaryInstallerItem |
||||
io.sc.platform.installer.item.FinishInstallerItem |
io.sc.platform.installer.item.FinishInstallerItem |
||||
@ -0,0 +1,91 @@ |
|||||
|
<!-- 协议 --> |
||||
|
<label for="protocol" class="form-label pt-2" th:text="#{ServerInstallerItem.protocol.label}">请求协议</label> |
||||
|
<select id="protocol" class="form-select form-select-sm" onchange="protocolChanged()"> |
||||
|
<option value="http" selected>HTTP</option> |
||||
|
<option value="https">HTTPS</option> |
||||
|
</select> |
||||
|
|
||||
|
<!-- IP地址 --> |
||||
|
<label for="ip" class="form-label pt-2" th:text="#{ServerInstallerItem.ip.label}">IP地址</label> |
||||
|
<input id="ip" type="text" class="form-control form-control-sm"> |
||||
|
|
||||
|
<!-- 端口 --> |
||||
|
<label for="port" class="form-label pt-2" th:text="#{ServerInstallerItem.port.label}">端口</label> |
||||
|
<input id="port" type="text" class="form-control form-control-sm" required value="8080"> |
||||
|
|
||||
|
<!-- web上下文 --> |
||||
|
<label for="webContextPath" class="form-label pt-2" th:text="#{ServerInstallerItem.webContextPath.label}">Web 上下文路径</label> |
||||
|
<input id="webContextPath" type="text" class="form-control form-control-sm" required value="/"> |
||||
|
|
||||
|
<!-- 下一步 --> |
||||
|
<div class="text-end pt-3"> |
||||
|
<button id="datasourcePreviousBtn" type="button" class="btn btn-outline-primary" th:text="#{io.sc.platform.installer.previousStep}" onclick="installer.previous()"></button> |
||||
|
<button id="datasourceNextBtn" type="button" class="btn btn-outline-primary" th:text="#{io.sc.platform.installer.nextStep}" onclick="datasourceNextStep()"></button> |
||||
|
</div> |
||||
|
|
||||
|
<script type="text/javascript"> |
||||
|
installer.register(new function(){ |
||||
|
this.getConfiguration =function(){ |
||||
|
if(document.getElementById('quickInstall').checked){ |
||||
|
return { |
||||
|
'ip' : '', |
||||
|
'port' : '8080', |
||||
|
'webContextPath' : '/' |
||||
|
}; |
||||
|
}else{ |
||||
|
var protocolIndex =document.getElementById('protocol').selectedIndex; |
||||
|
var protocol =document.getElementById('protocol').options[protocolIndex].value; |
||||
|
var ip =document.getElementById('ip').value; |
||||
|
var port =document.getElementById('port').value; |
||||
|
var webContextPath =document.getElementById('webContextPath').value; |
||||
|
|
||||
|
webContextPath =removeUrlPrefixSlash(webContextPath); |
||||
|
webContextPath =removeUrlSuffixSlash(webContextPath); |
||||
|
if(webContextPath){ |
||||
|
webContextPath ='/' + webContextPath + '/'; |
||||
|
}else{ |
||||
|
webContextPath ='/'; |
||||
|
} |
||||
|
|
||||
|
return { |
||||
|
'protocol' : protocol, |
||||
|
'ip' : ip, |
||||
|
'port' : port, |
||||
|
'webContextPath' : webContextPath |
||||
|
} |
||||
|
} |
||||
|
}; |
||||
|
}); |
||||
|
|
||||
|
function protocolChanged(){ |
||||
|
var protocol =document.getElementById("protocol").value; |
||||
|
if(protocol==='https'){ |
||||
|
document.getElementById("port").value='8443'; |
||||
|
}else { |
||||
|
document.getElementById("port").value='8080'; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
function removeUrlPrefixSlash(url){ |
||||
|
if (url) { |
||||
|
var _url = url; |
||||
|
while (_url.startsWith('/')) { |
||||
|
_url = _url.substring(1); |
||||
|
} |
||||
|
return _url; |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
|
||||
|
function removeUrlSuffixSlash(url) { |
||||
|
if (url) { |
||||
|
var _url = url; |
||||
|
while (_url.endsWith('/')) { |
||||
|
_url = _url.substring(0, _url.length - 1); |
||||
|
} |
||||
|
return _url; |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
|
</script> |
||||
@ -1,5 +1,6 @@ |
|||||
dependencies { |
dependencies { |
||||
api( |
api( |
||||
"com.dameng:DmJdbcDriver18:${jdbc_dm_version}", |
"com.dameng:DmJdbcDriver18:${jdbc_dm_version}", |
||||
|
"com.dameng:DmDialect-for-hibernate5.6:${dm_hibernate_version}" |
||||
) |
) |
||||
} |
} |
||||
@ -1,6 +1,6 @@ |
|||||
{ |
{ |
||||
"includes":[ |
"includes":[ |
||||
"io.sc.platform.jdbc.liquibase.controller", |
"io.sc.platform.jdbc.liquibase.installer.controller", |
||||
"io.sc.platform.jdbc.liquibase.service" |
"io.sc.platform.jdbc.liquibase.service" |
||||
] |
] |
||||
} |
} |
||||
@ -1,10 +1,10 @@ |
|||||
package io.sc.platform.job.core.vo; |
package io.sc.platform.job.core; |
||||
|
|
||||
import io.sc.platform.orm.api.vo.BaseVo; |
import io.sc.platform.orm.api.vo.BaseVo; |
||||
|
|
||||
import java.util.Date; |
import java.util.Date; |
||||
|
|
||||
public class TaskLogVo extends BaseVo { |
public class TaskLog extends BaseVo { |
||||
private String id; |
private String id; |
||||
private String executorId; |
private String executorId; |
||||
private String taskId; |
private String taskId; |
||||
@ -0,0 +1,24 @@ |
|||||
|
package io.sc.platform.job.core.exception; |
||||
|
|
||||
|
public class RcpException extends Exception { |
||||
|
public RcpException() { |
||||
|
super(); |
||||
|
} |
||||
|
|
||||
|
public RcpException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { |
||||
|
super(message, cause, enableSuppression, writableStackTrace); |
||||
|
} |
||||
|
|
||||
|
public RcpException(String message, Throwable cause) { |
||||
|
super(message, cause); |
||||
|
} |
||||
|
|
||||
|
public RcpException(String message) { |
||||
|
super(message); |
||||
|
} |
||||
|
|
||||
|
public RcpException(Throwable cause) { |
||||
|
super(cause); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -1,179 +0,0 @@ |
|||||
package io.sc.platform.job.core.thread; |
|
||||
|
|
||||
import com.xxl.job.admin.core.complete.XxlJobCompleter; |
|
||||
import com.xxl.job.admin.core.conf.XxlJobAdminConfig; |
|
||||
import com.xxl.job.admin.core.model.XxlJobLog; |
|
||||
import com.xxl.job.admin.core.util.I18nUtil; |
|
||||
import com.xxl.job.core.biz.model.HandleCallbackParam; |
|
||||
import com.xxl.job.core.biz.model.ReturnT; |
|
||||
import com.xxl.job.core.util.DateUtil; |
|
||||
import org.slf4j.Logger; |
|
||||
import org.slf4j.LoggerFactory; |
|
||||
|
|
||||
import java.util.Date; |
|
||||
import java.util.List; |
|
||||
import java.util.concurrent.*; |
|
||||
|
|
||||
public class JobCompleteHelper { |
|
||||
private static Logger logger = LoggerFactory.getLogger(JobCompleteHelper.class); |
|
||||
|
|
||||
private static JobCompleteHelper instance = new JobCompleteHelper(); |
|
||||
public static JobCompleteHelper getInstance(){ |
|
||||
return instance; |
|
||||
} |
|
||||
|
|
||||
// ---------------------- monitor ----------------------
|
|
||||
|
|
||||
private ThreadPoolExecutor callbackThreadPool = null; |
|
||||
private Thread monitorThread; |
|
||||
private volatile boolean toStop = false; |
|
||||
public void start(){ |
|
||||
|
|
||||
// for callback
|
|
||||
callbackThreadPool = new ThreadPoolExecutor( |
|
||||
2, |
|
||||
20, |
|
||||
30L, |
|
||||
TimeUnit.SECONDS, |
|
||||
new LinkedBlockingQueue<Runnable>(3000), |
|
||||
new ThreadFactory() { |
|
||||
@Override |
|
||||
public Thread newThread(Runnable r) { |
|
||||
return new Thread(r, "xxl-job, admin JobLosedMonitorHelper-callbackThreadPool-" + r.hashCode()); |
|
||||
} |
|
||||
}, |
|
||||
new RejectedExecutionHandler() { |
|
||||
@Override |
|
||||
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { |
|
||||
r.run(); |
|
||||
logger.warn(">>>>>>>>>>> xxl-job, callback too fast, match threadpool rejected handler(run now)."); |
|
||||
} |
|
||||
}); |
|
||||
|
|
||||
|
|
||||
// for monitor
|
|
||||
monitorThread = new Thread(new Runnable() { |
|
||||
|
|
||||
@Override |
|
||||
public void run() { |
|
||||
|
|
||||
// wait for JobTriggerPoolHelper-init
|
|
||||
try { |
|
||||
TimeUnit.MILLISECONDS.sleep(50); |
|
||||
} catch (InterruptedException e) { |
|
||||
if (!toStop) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// monitor
|
|
||||
while (!toStop) { |
|
||||
try { |
|
||||
// 任务结果丢失处理：调度记录停留在 "运行中" 状态超过10min，且对应执行器心跳注册失败不在线，则将本地调度主动标记失败；
|
|
||||
Date losedTime = DateUtil.addMinutes(new Date(), -10); |
|
||||
List<Long> losedJobIds = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().findLostJobIds(losedTime); |
|
||||
|
|
||||
if (losedJobIds!=null && losedJobIds.size()>0) { |
|
||||
for (Long logId: losedJobIds) { |
|
||||
|
|
||||
XxlJobLog jobLog = new XxlJobLog(); |
|
||||
jobLog.setId(logId); |
|
||||
|
|
||||
jobLog.setHandleTime(new Date()); |
|
||||
jobLog.setHandleCode(ReturnT.FAIL_CODE); |
|
||||
jobLog.setHandleMsg( I18nUtil.getString("joblog_lost_fail") ); |
|
||||
|
|
||||
XxlJobCompleter.updateHandleInfoAndFinish(jobLog); |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
} catch (Exception e) { |
|
||||
if (!toStop) { |
|
||||
logger.error(">>>>>>>>>>> xxl-job, job fail monitor thread error:{}", e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
try { |
|
||||
TimeUnit.SECONDS.sleep(60); |
|
||||
} catch (Exception e) { |
|
||||
if (!toStop) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
|
|
||||
logger.info(">>>>>>>>>>> xxl-job, JobLosedMonitorHelper stop"); |
|
||||
|
|
||||
} |
|
||||
}); |
|
||||
monitorThread.setDaemon(true); |
|
||||
monitorThread.setName("xxl-job, admin JobLosedMonitorHelper"); |
|
||||
monitorThread.start(); |
|
||||
} |
|
||||
|
|
||||
public void toStop(){ |
|
||||
toStop = true; |
|
||||
|
|
||||
// stop registryOrRemoveThreadPool
|
|
||||
callbackThreadPool.shutdownNow(); |
|
||||
|
|
||||
// stop monitorThread (interrupt and wait)
|
|
||||
monitorThread.interrupt(); |
|
||||
try { |
|
||||
monitorThread.join(); |
|
||||
} catch (InterruptedException e) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
|
|
||||
// ---------------------- helper ----------------------
|
|
||||
|
|
||||
public ReturnT<String> callback(List<HandleCallbackParam> callbackParamList) { |
|
||||
|
|
||||
callbackThreadPool.execute(new Runnable() { |
|
||||
@Override |
|
||||
public void run() { |
|
||||
for (HandleCallbackParam handleCallbackParam: callbackParamList) { |
|
||||
ReturnT<String> callbackResult = callback(handleCallbackParam); |
|
||||
logger.debug(">>>>>>>>> JobApiController.callback {}, handleCallbackParam={}, callbackResult={}", |
|
||||
(callbackResult.getCode()== ReturnT.SUCCESS_CODE?"success":"fail"), handleCallbackParam, callbackResult); |
|
||||
} |
|
||||
} |
|
||||
}); |
|
||||
|
|
||||
return ReturnT.SUCCESS; |
|
||||
} |
|
||||
|
|
||||
private ReturnT<String> callback(HandleCallbackParam handleCallbackParam) { |
|
||||
// valid log item
|
|
||||
XxlJobLog log = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().load(handleCallbackParam.getLogId()); |
|
||||
if (log == null) { |
|
||||
return new ReturnT<String>(ReturnT.FAIL_CODE, "log item not found."); |
|
||||
} |
|
||||
if (log.getHandleCode() > 0) { |
|
||||
return new ReturnT<String>(ReturnT.FAIL_CODE, "log repeate callback."); // avoid repeat callback, trigger child job etc
|
|
||||
} |
|
||||
|
|
||||
// handle msg
|
|
||||
StringBuffer handleMsg = new StringBuffer(); |
|
||||
if (log.getHandleMsg()!=null) { |
|
||||
handleMsg.append(log.getHandleMsg()).append("<br>"); |
|
||||
} |
|
||||
if (handleCallbackParam.getHandleMsg() != null) { |
|
||||
handleMsg.append(handleCallbackParam.getHandleMsg()); |
|
||||
} |
|
||||
|
|
||||
// success, save log
|
|
||||
log.setHandleTime(new Date()); |
|
||||
log.setHandleCode(handleCallbackParam.getHandleCode()); |
|
||||
log.setHandleMsg(handleMsg.toString()); |
|
||||
XxlJobCompleter.updateHandleInfoAndFinish(log); |
|
||||
|
|
||||
return ReturnT.SUCCESS; |
|
||||
} |
|
||||
|
|
||||
|
|
||||
|
|
||||
} |
|
||||
@ -1,110 +0,0 @@ |
|||||
package io.sc.platform.job.core.thread; |
|
||||
|
|
||||
import com.xxl.job.admin.core.conf.XxlJobAdminConfig; |
|
||||
import com.xxl.job.admin.core.model.XxlJobInfo; |
|
||||
import com.xxl.job.admin.core.model.XxlJobLog; |
|
||||
import com.xxl.job.admin.core.trigger.TriggerTypeEnum; |
|
||||
import com.xxl.job.admin.core.util.I18nUtil; |
|
||||
import org.slf4j.Logger; |
|
||||
import org.slf4j.LoggerFactory; |
|
||||
|
|
||||
import java.util.List; |
|
||||
import java.util.concurrent.TimeUnit; |
|
||||
|
|
||||
/** |
|
||||
* job monitor instance |
|
||||
* |
|
||||
* @author xuxueli 2015-9-1 18:05:56 |
|
||||
*/ |
|
||||
public class JobFailMonitorHelper { |
|
||||
private static Logger logger = LoggerFactory.getLogger(JobFailMonitorHelper.class); |
|
||||
|
|
||||
private static JobFailMonitorHelper instance = new JobFailMonitorHelper(); |
|
||||
public static JobFailMonitorHelper getInstance(){ |
|
||||
return instance; |
|
||||
} |
|
||||
|
|
||||
// ---------------------- monitor ----------------------
|
|
||||
|
|
||||
private Thread monitorThread; |
|
||||
private volatile boolean toStop = false; |
|
||||
public void start(){ |
|
||||
monitorThread = new Thread(new Runnable() { |
|
||||
|
|
||||
@Override |
|
||||
public void run() { |
|
||||
|
|
||||
// monitor
|
|
||||
while (!toStop) { |
|
||||
try { |
|
||||
|
|
||||
List<Long> failLogIds = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().findFailJobLogIds(1000); |
|
||||
if (failLogIds!=null && !failLogIds.isEmpty()) { |
|
||||
for (long failLogId: failLogIds) { |
|
||||
|
|
||||
// lock log
|
|
||||
int lockRet = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().updateAlarmStatus(failLogId, 0, -1); |
|
||||
if (lockRet < 1) { |
|
||||
continue; |
|
||||
} |
|
||||
XxlJobLog log = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().load(failLogId); |
|
||||
XxlJobInfo info = XxlJobAdminConfig.getAdminConfig().getXxlJobInfoDao().loadById(log.getJobId()); |
|
||||
|
|
||||
// 1、fail retry monitor
|
|
||||
if (log.getExecutorFailRetryCount() > 0) { |
|
||||
JobTriggerPoolHelper.trigger(log.getJobId(), TriggerTypeEnum.RETRY, (log.getExecutorFailRetryCount()-1), log.getExecutorShardingParam(), log.getExecutorParam(), null); |
|
||||
String retryMsg = "<br><br><span style=\"color:#F39C12;\" > >>>>>>>>>>>"+ I18nUtil.getString("jobconf_trigger_type_retry") +"<<<<<<<<<<< </span><br>"; |
|
||||
log.setTriggerMsg(log.getTriggerMsg() + retryMsg); |
|
||||
XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().updateTriggerInfo(log); |
|
||||
} |
|
||||
|
|
||||
// 2、fail alarm monitor
|
|
||||
int newAlarmStatus = 0; // 告警状态：0-默认、-1=锁定状态、1-无需告警、2-告警成功、3-告警失败
|
|
||||
if (info != null) { |
|
||||
boolean alarmResult = XxlJobAdminConfig.getAdminConfig().getJobAlarmer().alarm(info, log); |
|
||||
newAlarmStatus = alarmResult?2:3; |
|
||||
} else { |
|
||||
newAlarmStatus = 1; |
|
||||
} |
|
||||
|
|
||||
XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().updateAlarmStatus(failLogId, -1, newAlarmStatus); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
} catch (Exception e) { |
|
||||
if (!toStop) { |
|
||||
logger.error(">>>>>>>>>>> xxl-job, job fail monitor thread error:{}", e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
try { |
|
||||
TimeUnit.SECONDS.sleep(10); |
|
||||
} catch (Exception e) { |
|
||||
if (!toStop) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
|
|
||||
logger.info(">>>>>>>>>>> xxl-job, job fail monitor thread stop"); |
|
||||
|
|
||||
} |
|
||||
}); |
|
||||
monitorThread.setDaemon(true); |
|
||||
monitorThread.setName("xxl-job, admin JobFailMonitorHelper"); |
|
||||
monitorThread.start(); |
|
||||
} |
|
||||
|
|
||||
public void toStop(){ |
|
||||
toStop = true; |
|
||||
// interrupt and wait
|
|
||||
monitorThread.interrupt(); |
|
||||
try { |
|
||||
monitorThread.join(); |
|
||||
} catch (InterruptedException e) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
@ -1,152 +0,0 @@ |
|||||
package io.sc.platform.job.core.thread; |
|
||||
|
|
||||
import com.xxl.job.admin.core.conf.XxlJobAdminConfig; |
|
||||
import com.xxl.job.admin.core.model.XxlJobLogReport; |
|
||||
import org.slf4j.Logger; |
|
||||
import org.slf4j.LoggerFactory; |
|
||||
|
|
||||
import java.util.Calendar; |
|
||||
import java.util.Date; |
|
||||
import java.util.List; |
|
||||
import java.util.Map; |
|
||||
import java.util.concurrent.TimeUnit; |
|
||||
|
|
||||
/** |
|
||||
* job log report helper |
|
||||
* |
|
||||
* @author xuxueli 2019-11-22 |
|
||||
*/ |
|
||||
public class JobLogReportHelper { |
|
||||
private static Logger logger = LoggerFactory.getLogger(JobLogReportHelper.class); |
|
||||
|
|
||||
private static JobLogReportHelper instance = new JobLogReportHelper(); |
|
||||
public static JobLogReportHelper getInstance(){ |
|
||||
return instance; |
|
||||
} |
|
||||
|
|
||||
|
|
||||
private Thread logrThread; |
|
||||
private volatile boolean toStop = false; |
|
||||
public void start(){ |
|
||||
logrThread = new Thread(new Runnable() { |
|
||||
|
|
||||
@Override |
|
||||
public void run() { |
|
||||
|
|
||||
// last clean log time
|
|
||||
long lastCleanLogTime = 0; |
|
||||
|
|
||||
|
|
||||
while (!toStop) { |
|
||||
|
|
||||
// 1、log-report refresh: refresh log report in 3 days
|
|
||||
try { |
|
||||
|
|
||||
for (int i = 0; i < 3; i++) { |
|
||||
|
|
||||
// today
|
|
||||
Calendar itemDay = Calendar.getInstance(); |
|
||||
itemDay.add(Calendar.DAY_OF_MONTH, -i); |
|
||||
itemDay.set(Calendar.HOUR_OF_DAY, 0); |
|
||||
itemDay.set(Calendar.MINUTE, 0); |
|
||||
itemDay.set(Calendar.SECOND, 0); |
|
||||
itemDay.set(Calendar.MILLISECOND, 0); |
|
||||
|
|
||||
Date todayFrom = itemDay.getTime(); |
|
||||
|
|
||||
itemDay.set(Calendar.HOUR_OF_DAY, 23); |
|
||||
itemDay.set(Calendar.MINUTE, 59); |
|
||||
itemDay.set(Calendar.SECOND, 59); |
|
||||
itemDay.set(Calendar.MILLISECOND, 999); |
|
||||
|
|
||||
Date todayTo = itemDay.getTime(); |
|
||||
|
|
||||
// refresh log-report every minute
|
|
||||
XxlJobLogReport xxlJobLogReport = new XxlJobLogReport(); |
|
||||
xxlJobLogReport.setTriggerDay(todayFrom); |
|
||||
xxlJobLogReport.setRunningCount(0); |
|
||||
xxlJobLogReport.setSucCount(0); |
|
||||
xxlJobLogReport.setFailCount(0); |
|
||||
|
|
||||
Map<String, Object> triggerCountMap = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().findLogReport(todayFrom, todayTo); |
|
||||
if (triggerCountMap!=null && triggerCountMap.size()>0) { |
|
||||
int triggerDayCount = triggerCountMap.containsKey("triggerDayCount")?Integer.valueOf(String.valueOf(triggerCountMap.get("triggerDayCount"))):0; |
|
||||
int triggerDayCountRunning = triggerCountMap.containsKey("triggerDayCountRunning")?Integer.valueOf(String.valueOf(triggerCountMap.get("triggerDayCountRunning"))):0; |
|
||||
int triggerDayCountSuc = triggerCountMap.containsKey("triggerDayCountSuc")?Integer.valueOf(String.valueOf(triggerCountMap.get("triggerDayCountSuc"))):0; |
|
||||
int triggerDayCountFail = triggerDayCount - triggerDayCountRunning - triggerDayCountSuc; |
|
||||
|
|
||||
xxlJobLogReport.setRunningCount(triggerDayCountRunning); |
|
||||
xxlJobLogReport.setSucCount(triggerDayCountSuc); |
|
||||
xxlJobLogReport.setFailCount(triggerDayCountFail); |
|
||||
} |
|
||||
|
|
||||
// do refresh
|
|
||||
int ret = XxlJobAdminConfig.getAdminConfig().getXxlJobLogReportDao().update(xxlJobLogReport); |
|
||||
if (ret < 1) { |
|
||||
XxlJobAdminConfig.getAdminConfig().getXxlJobLogReportDao().save(xxlJobLogReport); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
} catch (Exception e) { |
|
||||
if (!toStop) { |
|
||||
logger.error(">>>>>>>>>>> xxl-job, job log report thread error:{}", e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// 2、log-clean: switch open & once each day
|
|
||||
if (XxlJobAdminConfig.getAdminConfig().getLogretentiondays()>0 |
|
||||
&& System.currentTimeMillis() - lastCleanLogTime > 24*60*60*1000) { |
|
||||
|
|
||||
// expire-time
|
|
||||
Calendar expiredDay = Calendar.getInstance(); |
|
||||
expiredDay.add(Calendar.DAY_OF_MONTH, -1 * XxlJobAdminConfig.getAdminConfig().getLogretentiondays()); |
|
||||
expiredDay.set(Calendar.HOUR_OF_DAY, 0); |
|
||||
expiredDay.set(Calendar.MINUTE, 0); |
|
||||
expiredDay.set(Calendar.SECOND, 0); |
|
||||
expiredDay.set(Calendar.MILLISECOND, 0); |
|
||||
Date clearBeforeTime = expiredDay.getTime(); |
|
||||
|
|
||||
// clean expired log
|
|
||||
List<Long> logIds = null; |
|
||||
do { |
|
||||
logIds = XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().findClearLogIds(0, 0, clearBeforeTime, 0, 1000); |
|
||||
if (logIds!=null && logIds.size()>0) { |
|
||||
XxlJobAdminConfig.getAdminConfig().getXxlJobLogDao().clearLog(logIds); |
|
||||
} |
|
||||
} while (logIds!=null && logIds.size()>0); |
|
||||
|
|
||||
// update clean time
|
|
||||
lastCleanLogTime = System.currentTimeMillis(); |
|
||||
} |
|
||||
|
|
||||
try { |
|
||||
TimeUnit.MINUTES.sleep(1); |
|
||||
} catch (Exception e) { |
|
||||
if (!toStop) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
|
|
||||
logger.info(">>>>>>>>>>> xxl-job, job log report thread stop"); |
|
||||
|
|
||||
} |
|
||||
}); |
|
||||
logrThread.setDaemon(true); |
|
||||
logrThread.setName("xxl-job, admin JobLogReportHelper"); |
|
||||
logrThread.start(); |
|
||||
} |
|
||||
|
|
||||
public void toStop(){ |
|
||||
toStop = true; |
|
||||
// interrupt and wait
|
|
||||
logrThread.interrupt(); |
|
||||
try { |
|
||||
logrThread.join(); |
|
||||
} catch (InterruptedException e) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
@ -1,204 +0,0 @@ |
|||||
package io.sc.platform.job.core.thread; |
|
||||
|
|
||||
import com.xxl.job.admin.core.conf.XxlJobAdminConfig; |
|
||||
import com.xxl.job.admin.core.model.XxlJobGroup; |
|
||||
import com.xxl.job.admin.core.model.XxlJobRegistry; |
|
||||
import com.xxl.job.core.biz.model.RegistryParam; |
|
||||
import com.xxl.job.core.biz.model.ReturnT; |
|
||||
import com.xxl.job.core.enums.RegistryConfig; |
|
||||
import org.slf4j.Logger; |
|
||||
import org.slf4j.LoggerFactory; |
|
||||
import org.springframework.util.StringUtils; |
|
||||
|
|
||||
import java.util.*; |
|
||||
import java.util.concurrent.*; |
|
||||
|
|
||||
/** |
|
||||
* job registry instance |
|
||||
* @author xuxueli 2016-10-02 19:10:24 |
|
||||
*/ |
|
||||
public class JobRegistryHelper { |
|
||||
private static Logger logger = LoggerFactory.getLogger(JobRegistryHelper.class); |
|
||||
|
|
||||
private static JobRegistryHelper instance = new JobRegistryHelper(); |
|
||||
public static JobRegistryHelper getInstance(){ |
|
||||
return instance; |
|
||||
} |
|
||||
|
|
||||
private ThreadPoolExecutor registryOrRemoveThreadPool = null; |
|
||||
private Thread registryMonitorThread; |
|
||||
private volatile boolean toStop = false; |
|
||||
|
|
||||
public void start(){ |
|
||||
|
|
||||
// for registry or remove
|
|
||||
registryOrRemoveThreadPool = new ThreadPoolExecutor( |
|
||||
2, |
|
||||
10, |
|
||||
30L, |
|
||||
TimeUnit.SECONDS, |
|
||||
new LinkedBlockingQueue<Runnable>(2000), |
|
||||
new ThreadFactory() { |
|
||||
@Override |
|
||||
public Thread newThread(Runnable r) { |
|
||||
return new Thread(r, "xxl-job, admin JobRegistryMonitorHelper-registryOrRemoveThreadPool-" + r.hashCode()); |
|
||||
} |
|
||||
}, |
|
||||
new RejectedExecutionHandler() { |
|
||||
@Override |
|
||||
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { |
|
||||
r.run(); |
|
||||
logger.warn(">>>>>>>>>>> xxl-job, registry or remove too fast, match threadpool rejected handler(run now)."); |
|
||||
} |
|
||||
}); |
|
||||
|
|
||||
// for monitor
|
|
||||
registryMonitorThread = new Thread(new Runnable() { |
|
||||
@Override |
|
||||
public void run() { |
|
||||
while (!toStop) { |
|
||||
try { |
|
||||
// auto registry group
|
|
||||
List<XxlJobGroup> groupList = XxlJobAdminConfig.getAdminConfig().getXxlJobGroupDao().findByAddressType(0); |
|
||||
if (groupList!=null && !groupList.isEmpty()) { |
|
||||
|
|
||||
// remove dead address (admin/executor)
|
|
||||
List<Integer> ids = XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().findDead(RegistryConfig.DEAD_TIMEOUT, new Date()); |
|
||||
if (ids!=null && ids.size()>0) { |
|
||||
XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().removeDead(ids); |
|
||||
} |
|
||||
|
|
||||
// fresh online address (admin/executor)
|
|
||||
HashMap<String, List<String>> appAddressMap = new HashMap<String, List<String>>(); |
|
||||
List<XxlJobRegistry> list = XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().findAll(RegistryConfig.DEAD_TIMEOUT, new Date()); |
|
||||
if (list != null) { |
|
||||
for (XxlJobRegistry item: list) { |
|
||||
if (RegistryConfig.RegistType.EXECUTOR.name().equals(item.getRegistryGroup())) { |
|
||||
String appname = item.getRegistryKey(); |
|
||||
List<String> registryList = appAddressMap.get(appname); |
|
||||
if (registryList == null) { |
|
||||
registryList = new ArrayList<String>(); |
|
||||
} |
|
||||
|
|
||||
if (!registryList.contains(item.getRegistryValue())) { |
|
||||
registryList.add(item.getRegistryValue()); |
|
||||
} |
|
||||
appAddressMap.put(appname, registryList); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// fresh group address
|
|
||||
for (XxlJobGroup group: groupList) { |
|
||||
List<String> registryList = appAddressMap.get(group.getAppname()); |
|
||||
String addressListStr = null; |
|
||||
if (registryList!=null && !registryList.isEmpty()) { |
|
||||
Collections.sort(registryList); |
|
||||
StringBuilder addressListSB = new StringBuilder(); |
|
||||
for (String item:registryList) { |
|
||||
addressListSB.append(item).append(","); |
|
||||
} |
|
||||
addressListStr = addressListSB.toString(); |
|
||||
addressListStr = addressListStr.substring(0, addressListStr.length()-1); |
|
||||
} |
|
||||
group.setAddressList(addressListStr); |
|
||||
group.setUpdateTime(new Date()); |
|
||||
|
|
||||
XxlJobAdminConfig.getAdminConfig().getXxlJobGroupDao().update(group); |
|
||||
} |
|
||||
} |
|
||||
} catch (Exception e) { |
|
||||
if (!toStop) { |
|
||||
logger.error(">>>>>>>>>>> xxl-job, job registry monitor thread error:{}", e); |
|
||||
} |
|
||||
} |
|
||||
try { |
|
||||
TimeUnit.SECONDS.sleep(RegistryConfig.BEAT_TIMEOUT); |
|
||||
} catch (InterruptedException e) { |
|
||||
if (!toStop) { |
|
||||
logger.error(">>>>>>>>>>> xxl-job, job registry monitor thread error:{}", e); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
logger.info(">>>>>>>>>>> xxl-job, job registry monitor thread stop"); |
|
||||
} |
|
||||
}); |
|
||||
registryMonitorThread.setDaemon(true); |
|
||||
registryMonitorThread.setName("xxl-job, admin JobRegistryMonitorHelper-registryMonitorThread"); |
|
||||
registryMonitorThread.start(); |
|
||||
} |
|
||||
|
|
||||
public void toStop(){ |
|
||||
toStop = true; |
|
||||
|
|
||||
// stop registryOrRemoveThreadPool
|
|
||||
registryOrRemoveThreadPool.shutdownNow(); |
|
||||
|
|
||||
// stop monitir (interrupt and wait)
|
|
||||
registryMonitorThread.interrupt(); |
|
||||
try { |
|
||||
registryMonitorThread.join(); |
|
||||
} catch (InterruptedException e) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
|
|
||||
// ---------------------- helper ----------------------
|
|
||||
|
|
||||
public ReturnT<String> registry(RegistryParam registryParam) { |
|
||||
|
|
||||
// valid
|
|
||||
if (!StringUtils.hasText(registryParam.getRegistryGroup()) |
|
||||
|| !StringUtils.hasText(registryParam.getRegistryKey()) |
|
||||
|| !StringUtils.hasText(registryParam.getRegistryValue())) { |
|
||||
return new ReturnT<String>(ReturnT.FAIL_CODE, "Illegal Argument."); |
|
||||
} |
|
||||
|
|
||||
// async execute
|
|
||||
registryOrRemoveThreadPool.execute(new Runnable() { |
|
||||
@Override |
|
||||
public void run() { |
|
||||
int ret = XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().registryUpdate(registryParam.getRegistryGroup(), registryParam.getRegistryKey(), registryParam.getRegistryValue(), new Date()); |
|
||||
if (ret < 1) { |
|
||||
XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().registrySave(registryParam.getRegistryGroup(), registryParam.getRegistryKey(), registryParam.getRegistryValue(), new Date()); |
|
||||
|
|
||||
// fresh
|
|
||||
freshGroupRegistryInfo(registryParam); |
|
||||
} |
|
||||
} |
|
||||
}); |
|
||||
|
|
||||
return ReturnT.SUCCESS; |
|
||||
} |
|
||||
|
|
||||
public ReturnT<String> registryRemove(RegistryParam registryParam) { |
|
||||
|
|
||||
// valid
|
|
||||
if (!StringUtils.hasText(registryParam.getRegistryGroup()) |
|
||||
|| !StringUtils.hasText(registryParam.getRegistryKey()) |
|
||||
|| !StringUtils.hasText(registryParam.getRegistryValue())) { |
|
||||
return new ReturnT<String>(ReturnT.FAIL_CODE, "Illegal Argument."); |
|
||||
} |
|
||||
|
|
||||
// async execute
|
|
||||
registryOrRemoveThreadPool.execute(new Runnable() { |
|
||||
@Override |
|
||||
public void run() { |
|
||||
int ret = XxlJobAdminConfig.getAdminConfig().getXxlJobRegistryDao().registryDelete(registryParam.getRegistryGroup(), registryParam.getRegistryKey(), registryParam.getRegistryValue()); |
|
||||
if (ret > 0) { |
|
||||
// fresh
|
|
||||
freshGroupRegistryInfo(registryParam); |
|
||||
} |
|
||||
} |
|
||||
}); |
|
||||
|
|
||||
return ReturnT.SUCCESS; |
|
||||
} |
|
||||
|
|
||||
private void freshGroupRegistryInfo(RegistryParam registryParam){ |
|
||||
// Under consideration, prevent affecting core tables
|
|
||||
} |
|
||||
|
|
||||
|
|
||||
} |
|
||||
@ -1,368 +0,0 @@ |
|||||
package io.sc.platform.job.core.thread; |
|
||||
|
|
||||
import com.xxl.job.admin.core.conf.XxlJobAdminConfig; |
|
||||
import com.xxl.job.admin.core.cron.CronExpression; |
|
||||
import com.xxl.job.admin.core.model.XxlJobInfo; |
|
||||
import com.xxl.job.admin.core.scheduler.MisfireStrategyEnum; |
|
||||
import com.xxl.job.admin.core.scheduler.ScheduleTypeEnum; |
|
||||
import com.xxl.job.admin.core.trigger.TriggerTypeEnum; |
|
||||
import io.sc.platform.core.Environment; |
|
||||
import org.slf4j.Logger; |
|
||||
import org.slf4j.LoggerFactory; |
|
||||
|
|
||||
import java.sql.Connection; |
|
||||
import java.sql.PreparedStatement; |
|
||||
import java.sql.SQLException; |
|
||||
import java.util.*; |
|
||||
import java.util.concurrent.ConcurrentHashMap; |
|
||||
import java.util.concurrent.TimeUnit; |
|
||||
|
|
||||
public class JobScheduleHelper { |
|
||||
private static Logger logger = LoggerFactory.getLogger(JobScheduleHelper.class); |
|
||||
|
|
||||
public static final long PRE_READ_MS = 5000; // pre read
|
|
||||
private Thread scheduleThread; |
|
||||
private Thread ringThread; |
|
||||
private volatile boolean scheduleThreadToStop = false; |
|
||||
private volatile boolean ringThreadToStop = false; |
|
||||
private volatile static Map<Integer, List<Integer>> ringData = new ConcurrentHashMap<>(); |
|
||||
|
|
||||
private static class JobScheduleHelperHolder{ |
|
||||
private static JobScheduleHelper instance =new JobScheduleHelper(); |
|
||||
} |
|
||||
|
|
||||
private JobScheduleHelper(){} |
|
||||
public static JobScheduleHelper getInstance(){ |
|
||||
return JobScheduleHelperHolder.instance; |
|
||||
} |
|
||||
|
|
||||
public void start(){ |
|
||||
// schedule thread
|
|
||||
scheduleThread = new Thread(new Runnable() { |
|
||||
@Override |
|
||||
public void run() { |
|
||||
try { |
|
||||
TimeUnit.MILLISECONDS.sleep(5000 - System.currentTimeMillis()%1000 ); |
|
||||
} catch (InterruptedException e) { |
|
||||
if (!scheduleThreadToStop) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
logger.info(">>>>>>>>> init xxl-job admin scheduler success."); |
|
||||
|
|
||||
// pre-read count: treadpool-size * trigger-qps (each trigger cost 50ms, qps = 1000/50 = 20)
|
|
||||
int preReadCount = (XxlJobAdminConfig.getAdminConfig().getTriggerPoolFastMax() + XxlJobAdminConfig.getAdminConfig().getTriggerPoolSlowMax()) * 20; |
|
||||
|
|
||||
while (!scheduleThreadToStop) { |
|
||||
|
|
||||
// Scan Job
|
|
||||
long start = System.currentTimeMillis(); |
|
||||
|
|
||||
Connection conn = null; |
|
||||
Boolean connAutoCommit = null; |
|
||||
PreparedStatement preparedStatement = null; |
|
||||
|
|
||||
boolean preReadSuc = true; |
|
||||
try { |
|
||||
|
|
||||
conn = XxlJobAdminConfig.getAdminConfig().getDataSource().getConnection(); |
|
||||
connAutoCommit = conn.getAutoCommit(); |
|
||||
conn.setAutoCommit(false); |
|
||||
|
|
||||
preparedStatement = conn.prepareStatement( "select * from xxl_job_lock where lock_name = 'schedule_lock' for update" ); |
|
||||
preparedStatement.execute(); |
|
||||
|
|
||||
// tx start
|
|
||||
|
|
||||
// 1、pre read
|
|
||||
long nowTime = System.currentTimeMillis(); |
|
||||
List<XxlJobInfo> scheduleList = XxlJobAdminConfig.getAdminConfig().getXxlJobInfoDao().scheduleJobQuery(nowTime + PRE_READ_MS, preReadCount); |
|
||||
if (scheduleList!=null && scheduleList.size()>0) { |
|
||||
// 2、push time-ring
|
|
||||
for (XxlJobInfo jobInfo: scheduleList) { |
|
||||
|
|
||||
// time-ring jump
|
|
||||
if (nowTime > jobInfo.getTriggerNextTime() + PRE_READ_MS) { |
|
||||
// 2.1、trigger-expire > 5s：pass && make next-trigger-time
|
|
||||
logger.warn(">>>>>>>>>>> xxl-job, schedule misfire, jobId = " + jobInfo.getId()); |
|
||||
|
|
||||
// 1、misfire match
|
|
||||
MisfireStrategyEnum misfireStrategyEnum = MisfireStrategyEnum.match(jobInfo.getMisfireStrategy(), MisfireStrategyEnum.DO_NOTHING); |
|
||||
if (MisfireStrategyEnum.FIRE_ONCE_NOW == misfireStrategyEnum) { |
|
||||
// FIRE_ONCE_NOW 》 trigger
|
|
||||
JobTriggerPoolHelper.trigger(jobInfo.getId(), TriggerTypeEnum.MISFIRE, -1, null, null, null); |
|
||||
logger.debug(">>>>>>>>>>> xxl-job, schedule push trigger : jobId = " + jobInfo.getId() ); |
|
||||
} |
|
||||
|
|
||||
// 2、fresh next
|
|
||||
refreshNextValidTime(jobInfo, new Date()); |
|
||||
|
|
||||
} else if (nowTime > jobInfo.getTriggerNextTime()) { |
|
||||
// 2.2、trigger-expire < 5s：direct-trigger && make next-trigger-time
|
|
||||
|
|
||||
// 1、trigger
|
|
||||
JobTriggerPoolHelper.trigger(jobInfo.getId(), TriggerTypeEnum.CRON, -1, null, null, null); |
|
||||
logger.debug(">>>>>>>>>>> xxl-job, schedule push trigger : jobId = " + jobInfo.getId() ); |
|
||||
|
|
||||
// 2、fresh next
|
|
||||
refreshNextValidTime(jobInfo, new Date()); |
|
||||
|
|
||||
// next-trigger-time in 5s, pre-read again
|
|
||||
if (jobInfo.getTriggerStatus()==1 && nowTime + PRE_READ_MS > jobInfo.getTriggerNextTime()) { |
|
||||
|
|
||||
// 1、make ring second
|
|
||||
int ringSecond = (int)((jobInfo.getTriggerNextTime()/1000)%60); |
|
||||
|
|
||||
// 2、push time ring
|
|
||||
pushTimeRing(ringSecond, jobInfo.getId()); |
|
||||
|
|
||||
// 3、fresh next
|
|
||||
refreshNextValidTime(jobInfo, new Date(jobInfo.getTriggerNextTime())); |
|
||||
|
|
||||
} |
|
||||
|
|
||||
} else { |
|
||||
// 2.3、trigger-pre-read：time-ring trigger && make next-trigger-time
|
|
||||
|
|
||||
// 1、make ring second
|
|
||||
int ringSecond = (int)((jobInfo.getTriggerNextTime()/1000)%60); |
|
||||
|
|
||||
// 2、push time ring
|
|
||||
pushTimeRing(ringSecond, jobInfo.getId()); |
|
||||
|
|
||||
// 3、fresh next
|
|
||||
refreshNextValidTime(jobInfo, new Date(jobInfo.getTriggerNextTime())); |
|
||||
|
|
||||
} |
|
||||
|
|
||||
} |
|
||||
|
|
||||
// 3、update trigger info
|
|
||||
for (XxlJobInfo jobInfo: scheduleList) { |
|
||||
XxlJobAdminConfig.getAdminConfig().getXxlJobInfoDao().scheduleUpdate(jobInfo); |
|
||||
} |
|
||||
|
|
||||
} else { |
|
||||
preReadSuc = false; |
|
||||
} |
|
||||
|
|
||||
// tx stop
|
|
||||
|
|
||||
|
|
||||
} catch (Exception e) { |
|
||||
if (!scheduleThreadToStop) { |
|
||||
logger.error(">>>>>>>>>>> xxl-job, JobScheduleHelper#scheduleThread error:{}", e); |
|
||||
} |
|
||||
} finally { |
|
||||
|
|
||||
// commit
|
|
||||
if (conn != null) { |
|
||||
try { |
|
||||
conn.commit(); |
|
||||
} catch (SQLException e) { |
|
||||
if (!scheduleThreadToStop) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
try { |
|
||||
conn.setAutoCommit(connAutoCommit); |
|
||||
} catch (SQLException e) { |
|
||||
if (!scheduleThreadToStop) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
try { |
|
||||
conn.close(); |
|
||||
} catch (SQLException e) { |
|
||||
if (!scheduleThreadToStop) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// close PreparedStatement
|
|
||||
if (null != preparedStatement) { |
|
||||
try { |
|
||||
preparedStatement.close(); |
|
||||
} catch (SQLException e) { |
|
||||
if (!scheduleThreadToStop) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
long cost = System.currentTimeMillis()-start; |
|
||||
|
|
||||
|
|
||||
// Wait seconds, align second
|
|
||||
if (cost < 1000) { // scan-overtime, not wait
|
|
||||
try { |
|
||||
// pre-read period: success > scan each second; fail > skip this period;
|
|
||||
TimeUnit.MILLISECONDS.sleep((preReadSuc?1000:PRE_READ_MS) - System.currentTimeMillis()%1000); |
|
||||
} catch (InterruptedException e) { |
|
||||
if (!scheduleThreadToStop) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
|
|
||||
logger.info(">>>>>>>>>>> xxl-job, JobScheduleHelper#scheduleThread stop"); |
|
||||
} |
|
||||
}); |
|
||||
scheduleThread.setDaemon(true); |
|
||||
scheduleThread.setName("xxl-job, admin JobScheduleHelper#scheduleThread"); |
|
||||
scheduleThread.start(); |
|
||||
|
|
||||
|
|
||||
// ring thread
|
|
||||
ringThread = new Thread(new Runnable() { |
|
||||
@Override |
|
||||
public void run() { |
|
||||
|
|
||||
while (!ringThreadToStop) { |
|
||||
|
|
||||
// align second
|
|
||||
try { |
|
||||
TimeUnit.MILLISECONDS.sleep(1000 - System.currentTimeMillis() % 1000); |
|
||||
} catch (InterruptedException e) { |
|
||||
if (!ringThreadToStop) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
try { |
|
||||
// second data
|
|
||||
List<Integer> ringItemData = new ArrayList<>(); |
|
||||
int nowSecond = Calendar.getInstance().get(Calendar.SECOND); // 避免处理耗时太长，跨过刻度，向前校验一个刻度；
|
|
||||
for (int i = 0; i < 2; i++) { |
|
||||
List<Integer> tmpData = ringData.remove( (nowSecond+60-i)%60 ); |
|
||||
if (tmpData != null) { |
|
||||
ringItemData.addAll(tmpData); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// ring trigger
|
|
||||
logger.debug(">>>>>>>>>>> xxl-job, time-ring beat : " + nowSecond + " = " + Arrays.asList(ringItemData) ); |
|
||||
if (ringItemData.size() > 0) { |
|
||||
// do trigger
|
|
||||
for (int jobId: ringItemData) { |
|
||||
// do trigger
|
|
||||
JobTriggerPoolHelper.trigger(jobId, TriggerTypeEnum.CRON, -1, null, null, null); |
|
||||
} |
|
||||
// clear
|
|
||||
ringItemData.clear(); |
|
||||
} |
|
||||
} catch (Exception e) { |
|
||||
if (!ringThreadToStop) { |
|
||||
logger.error(">>>>>>>>>>> xxl-job, JobScheduleHelper#ringThread error:{}", e); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
logger.info(">>>>>>>>>>> xxl-job, JobScheduleHelper#ringThread stop"); |
|
||||
} |
|
||||
}); |
|
||||
ringThread.setDaemon(true); |
|
||||
ringThread.setName("xxl-job, admin JobScheduleHelper#ringThread"); |
|
||||
ringThread.start(); |
|
||||
} |
|
||||
|
|
||||
private void refreshNextValidTime(XxlJobInfo jobInfo, Date fromTime) throws Exception { |
|
||||
Date nextValidTime = generateNextValidTime(jobInfo, fromTime); |
|
||||
if (nextValidTime != null) { |
|
||||
jobInfo.setTriggerLastTime(jobInfo.getTriggerNextTime()); |
|
||||
jobInfo.setTriggerNextTime(nextValidTime.getTime()); |
|
||||
} else { |
|
||||
jobInfo.setTriggerStatus(0); |
|
||||
jobInfo.setTriggerLastTime(0); |
|
||||
jobInfo.setTriggerNextTime(0); |
|
||||
logger.warn(">>>>>>>>>>> xxl-job, refreshNextValidTime fail for job: jobId={}, scheduleType={}, scheduleConf={}", |
|
||||
jobInfo.getId(), jobInfo.getScheduleType(), jobInfo.getScheduleConf()); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
private void pushTimeRing(int ringSecond, int jobId){ |
|
||||
// push async ring
|
|
||||
List<Integer> ringItemData = ringData.get(ringSecond); |
|
||||
if (ringItemData == null) { |
|
||||
ringItemData = new ArrayList<Integer>(); |
|
||||
ringData.put(ringSecond, ringItemData); |
|
||||
} |
|
||||
ringItemData.add(jobId); |
|
||||
|
|
||||
logger.debug(">>>>>>>>>>> xxl-job, schedule push time-ring : " + ringSecond + " = " + Arrays.asList(ringItemData) ); |
|
||||
} |
|
||||
|
|
||||
public void toStop(){ |
|
||||
|
|
||||
// 1、stop schedule
|
|
||||
scheduleThreadToStop = true; |
|
||||
try { |
|
||||
TimeUnit.SECONDS.sleep(1); // wait
|
|
||||
} catch (InterruptedException e) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
if (scheduleThread.getState() != Thread.State.TERMINATED){ |
|
||||
// interrupt and wait
|
|
||||
scheduleThread.interrupt(); |
|
||||
try { |
|
||||
scheduleThread.join(); |
|
||||
} catch (InterruptedException e) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// if has ring data
|
|
||||
boolean hasRingData = false; |
|
||||
if (!ringData.isEmpty()) { |
|
||||
for (int second : ringData.keySet()) { |
|
||||
List<Integer> tmpData = ringData.get(second); |
|
||||
if (tmpData!=null && tmpData.size()>0) { |
|
||||
hasRingData = true; |
|
||||
break; |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
if (hasRingData) { |
|
||||
try { |
|
||||
TimeUnit.SECONDS.sleep(8); |
|
||||
} catch (InterruptedException e) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
// stop ring (wait job-in-memory stop)
|
|
||||
ringThreadToStop = true; |
|
||||
try { |
|
||||
TimeUnit.SECONDS.sleep(1); |
|
||||
} catch (InterruptedException e) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
if (ringThread.getState() != Thread.State.TERMINATED){ |
|
||||
// interrupt and wait
|
|
||||
ringThread.interrupt(); |
|
||||
try { |
|
||||
ringThread.join(); |
|
||||
} catch (InterruptedException e) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
logger.info(">>>>>>>>>>> xxl-job, JobScheduleHelper stop"); |
|
||||
} |
|
||||
|
|
||||
|
|
||||
// ---------------------- tools ----------------------
|
|
||||
public static Date generateNextValidTime(XxlJobInfo jobInfo, Date fromTime) throws Exception { |
|
||||
ScheduleTypeEnum scheduleTypeEnum = ScheduleTypeEnum.match(jobInfo.getScheduleType(), null); |
|
||||
if (ScheduleTypeEnum.CRON == scheduleTypeEnum) { |
|
||||
Date nextValidTime = new CronExpression(jobInfo.getScheduleConf()).getNextValidTimeAfter(fromTime); |
|
||||
return nextValidTime; |
|
||||
} else if (ScheduleTypeEnum.FIX_RATE == scheduleTypeEnum /*|| ScheduleTypeEnum.FIX_DELAY == scheduleTypeEnum*/) { |
|
||||
return new Date(fromTime.getTime() + Integer.valueOf(jobInfo.getScheduleConf())*1000 ); |
|
||||
} |
|
||||
return null; |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
@ -1,150 +0,0 @@ |
|||||
package io.sc.platform.job.core.thread; |
|
||||
|
|
||||
import com.xxl.job.admin.core.conf.XxlJobAdminConfig; |
|
||||
import com.xxl.job.admin.core.trigger.TriggerTypeEnum; |
|
||||
import com.xxl.job.admin.core.trigger.XxlJobTrigger; |
|
||||
import org.slf4j.Logger; |
|
||||
import org.slf4j.LoggerFactory; |
|
||||
|
|
||||
import java.util.concurrent.*; |
|
||||
import java.util.concurrent.atomic.AtomicInteger; |
|
||||
|
|
||||
/** |
|
||||
* job trigger thread pool helper |
|
||||
* |
|
||||
* @author xuxueli 2018-07-03 21:08:07 |
|
||||
*/ |
|
||||
public class JobTriggerPoolHelper { |
|
||||
private static Logger logger = LoggerFactory.getLogger(JobTriggerPoolHelper.class); |
|
||||
|
|
||||
|
|
||||
// ---------------------- trigger pool ----------------------
|
|
||||
|
|
||||
// fast/slow thread pool
|
|
||||
private ThreadPoolExecutor fastTriggerPool = null; |
|
||||
private ThreadPoolExecutor slowTriggerPool = null; |
|
||||
|
|
||||
public void start(){ |
|
||||
fastTriggerPool = new ThreadPoolExecutor( |
|
||||
10, |
|
||||
XxlJobAdminConfig.getAdminConfig().getTriggerPoolFastMax(), |
|
||||
60L, |
|
||||
TimeUnit.SECONDS, |
|
||||
new LinkedBlockingQueue<Runnable>(1000), |
|
||||
new ThreadFactory() { |
|
||||
@Override |
|
||||
public Thread newThread(Runnable r) { |
|
||||
return new Thread(r, "xxl-job, admin JobTriggerPoolHelper-fastTriggerPool-" + r.hashCode()); |
|
||||
} |
|
||||
}); |
|
||||
|
|
||||
slowTriggerPool = new ThreadPoolExecutor( |
|
||||
10, |
|
||||
XxlJobAdminConfig.getAdminConfig().getTriggerPoolSlowMax(), |
|
||||
60L, |
|
||||
TimeUnit.SECONDS, |
|
||||
new LinkedBlockingQueue<Runnable>(2000), |
|
||||
new ThreadFactory() { |
|
||||
@Override |
|
||||
public Thread newThread(Runnable r) { |
|
||||
return new Thread(r, "xxl-job, admin JobTriggerPoolHelper-slowTriggerPool-" + r.hashCode()); |
|
||||
} |
|
||||
}); |
|
||||
} |
|
||||
|
|
||||
|
|
||||
public void stop() { |
|
||||
//triggerPool.shutdown();
|
|
||||
fastTriggerPool.shutdownNow(); |
|
||||
slowTriggerPool.shutdownNow(); |
|
||||
logger.info(">>>>>>>>> xxl-job trigger thread pool shutdown success."); |
|
||||
} |
|
||||
|
|
||||
|
|
||||
// job timeout count
|
|
||||
private volatile long minTim = System.currentTimeMillis()/60000; // ms > min
|
|
||||
private volatile ConcurrentMap<Integer, AtomicInteger> jobTimeoutCountMap = new ConcurrentHashMap<>(); |
|
||||
|
|
||||
|
|
||||
/** |
|
||||
* add trigger |
|
||||
*/ |
|
||||
public void addTrigger(final int jobId, |
|
||||
final TriggerTypeEnum triggerType, |
|
||||
final int failRetryCount, |
|
||||
final String executorShardingParam, |
|
||||
final String executorParam, |
|
||||
final String addressList) { |
|
||||
|
|
||||
// choose thread pool
|
|
||||
ThreadPoolExecutor triggerPool_ = fastTriggerPool; |
|
||||
AtomicInteger jobTimeoutCount = jobTimeoutCountMap.get(jobId); |
|
||||
if (jobTimeoutCount!=null && jobTimeoutCount.get() > 10) { // job-timeout 10 times in 1 min
|
|
||||
triggerPool_ = slowTriggerPool; |
|
||||
} |
|
||||
|
|
||||
// trigger
|
|
||||
triggerPool_.execute(new Runnable() { |
|
||||
@Override |
|
||||
public void run() { |
|
||||
|
|
||||
long start = System.currentTimeMillis(); |
|
||||
|
|
||||
try { |
|
||||
// do trigger
|
|
||||
XxlJobTrigger.trigger(jobId, triggerType, failRetryCount, executorShardingParam, executorParam, addressList); |
|
||||
} catch (Exception e) { |
|
||||
logger.error(e.getMessage(), e); |
|
||||
} finally { |
|
||||
|
|
||||
// check timeout-count-map
|
|
||||
long minTim_now = System.currentTimeMillis()/60000; |
|
||||
if (minTim != minTim_now) { |
|
||||
minTim = minTim_now; |
|
||||
jobTimeoutCountMap.clear(); |
|
||||
} |
|
||||
|
|
||||
// incr timeout-count-map
|
|
||||
long cost = System.currentTimeMillis()-start; |
|
||||
if (cost > 500) { // ob-timeout threshold 500ms
|
|
||||
AtomicInteger timeoutCount = jobTimeoutCountMap.putIfAbsent(jobId, new AtomicInteger(1)); |
|
||||
if (timeoutCount != null) { |
|
||||
timeoutCount.incrementAndGet(); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
|
|
||||
} |
|
||||
}); |
|
||||
} |
|
||||
|
|
||||
|
|
||||
|
|
||||
// ---------------------- helper ----------------------
|
|
||||
|
|
||||
private static JobTriggerPoolHelper helper = new JobTriggerPoolHelper(); |
|
||||
|
|
||||
public static void toStart() { |
|
||||
helper.start(); |
|
||||
} |
|
||||
public static void toStop() { |
|
||||
helper.stop(); |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* @param jobId |
|
||||
* @param triggerType |
|
||||
* @param failRetryCount |
|
||||
* >=0: use this param |
|
||||
* <0: use param from job info config |
|
||||
* @param executorShardingParam |
|
||||
* @param executorParam |
|
||||
* null: use job param |
|
||||
* not null: cover job param |
|
||||
*/ |
|
||||
public static void trigger(int jobId, TriggerTypeEnum triggerType, int failRetryCount, String executorShardingParam, String executorParam, String addressList) { |
|
||||
helper.addTrigger(jobId, triggerType, failRetryCount, executorShardingParam, executorParam, addressList); |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
@ -0,0 +1,5 @@ |
|||||
|
dependencies { |
||||
|
api( |
||||
|
project(":io.sc.platform.job.core"), |
||||
|
) |
||||
|
} |
||||
@ -0,0 +1,45 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
/** |
||||
|
* @author xuxueli 2017-07-27 21:52:49 |
||||
|
*/ |
||||
|
public interface AdminBiz { |
||||
|
|
||||
|
|
||||
|
// ---------------------- callback ----------------------
|
||||
|
|
||||
|
/** |
||||
|
* callback |
||||
|
* |
||||
|
* @param callbackParamList |
||||
|
* @return |
||||
|
*/ |
||||
|
public ReturnT<String> callback(List<HandleCallbackParam> callbackParamList); |
||||
|
|
||||
|
|
||||
|
// ---------------------- registry ----------------------
|
||||
|
|
||||
|
/** |
||||
|
* registry |
||||
|
* |
||||
|
* @param registryParam |
||||
|
* @return |
||||
|
*/ |
||||
|
public ReturnT<String> registry(RegistryParam registryParam); |
||||
|
|
||||
|
/** |
||||
|
* registry remove |
||||
|
* |
||||
|
* @param registryParam |
||||
|
* @return |
||||
|
*/ |
||||
|
public ReturnT<String> registryRemove(RegistryParam registryParam); |
||||
|
|
||||
|
|
||||
|
// ---------------------- biz (custome) ----------------------
|
||||
|
// group、job ... manage
|
||||
|
|
||||
|
} |
||||
@ -0,0 +1,47 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
|
||||
|
import io.sc.platform.job.executor.util.XxlJobRemotingUtil; |
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
/** |
||||
|
* admin api test |
||||
|
* |
||||
|
* @author xuxueli 2017-07-28 22:14:52 |
||||
|
*/ |
||||
|
public class AdminBizClient implements AdminBiz { |
||||
|
|
||||
|
public AdminBizClient() { |
||||
|
} |
||||
|
public AdminBizClient(String addressUrl, String accessToken) { |
||||
|
this.addressUrl = addressUrl; |
||||
|
this.accessToken = accessToken; |
||||
|
|
||||
|
// valid
|
||||
|
if (!this.addressUrl.endsWith("/")) { |
||||
|
this.addressUrl = this.addressUrl + "/"; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private String addressUrl ; |
||||
|
private String accessToken; |
||||
|
private int timeout = 3; |
||||
|
|
||||
|
|
||||
|
@Override |
||||
|
public ReturnT<String> callback(List<HandleCallbackParam> callbackParamList) { |
||||
|
return XxlJobRemotingUtil.postBody(addressUrl+"api/callback", accessToken, timeout, callbackParamList, String.class); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public ReturnT<String> registry(RegistryParam registryParam) { |
||||
|
return XxlJobRemotingUtil.postBody(addressUrl + "api/registry", accessToken, timeout, registryParam, String.class); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public ReturnT<String> registryRemove(RegistryParam registryParam) { |
||||
|
return XxlJobRemotingUtil.postBody(addressUrl + "api/registryRemove", accessToken, timeout, registryParam, String.class); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,64 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
import java.io.Serializable; |
||||
|
|
||||
|
public class HandleCallbackParam implements Serializable { |
||||
|
private static final long serialVersionUID = 42L; |
||||
|
|
||||
|
private long logId; |
||||
|
private long logDateTim; |
||||
|
|
||||
|
private int handleCode; |
||||
|
private String handleMsg; |
||||
|
|
||||
|
public HandleCallbackParam(){} |
||||
|
public HandleCallbackParam(long logId, long logDateTim, int handleCode, String handleMsg) { |
||||
|
this.logId = logId; |
||||
|
this.logDateTim = logDateTim; |
||||
|
this.handleCode = handleCode; |
||||
|
this.handleMsg = handleMsg; |
||||
|
} |
||||
|
|
||||
|
public long getLogId() { |
||||
|
return logId; |
||||
|
} |
||||
|
|
||||
|
public void setLogId(long logId) { |
||||
|
this.logId = logId; |
||||
|
} |
||||
|
|
||||
|
public long getLogDateTim() { |
||||
|
return logDateTim; |
||||
|
} |
||||
|
|
||||
|
public void setLogDateTim(long logDateTim) { |
||||
|
this.logDateTim = logDateTim; |
||||
|
} |
||||
|
|
||||
|
public int getHandleCode() { |
||||
|
return handleCode; |
||||
|
} |
||||
|
|
||||
|
public void setHandleCode(int handleCode) { |
||||
|
this.handleCode = handleCode; |
||||
|
} |
||||
|
|
||||
|
public String getHandleMsg() { |
||||
|
return handleMsg; |
||||
|
} |
||||
|
|
||||
|
public void setHandleMsg(String handleMsg) { |
||||
|
this.handleMsg = handleMsg; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public String toString() { |
||||
|
return "HandleCallbackParam{" + |
||||
|
"logId=" + logId + |
||||
|
", logDateTim=" + logDateTim + |
||||
|
", handleCode=" + handleCode + |
||||
|
", handleMsg='" + handleMsg + '\'' + |
||||
|
'}'; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,11 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
public abstract class JobHandler { |
||||
|
public abstract void execute() throws Exception; |
||||
|
|
||||
|
public void init() throws Exception { |
||||
|
} |
||||
|
|
||||
|
public void destroy() throws Exception { |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,56 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
import java.io.Serializable; |
||||
|
|
||||
|
/** |
||||
|
* Created by xuxueli on 17/3/23. |
||||
|
*/ |
||||
|
public class LogResult implements Serializable { |
||||
|
private static final long serialVersionUID = 42L; |
||||
|
|
||||
|
public LogResult() { |
||||
|
} |
||||
|
public LogResult(int fromLineNum, int toLineNum, String logContent, boolean isEnd) { |
||||
|
this.fromLineNum = fromLineNum; |
||||
|
this.toLineNum = toLineNum; |
||||
|
this.logContent = logContent; |
||||
|
this.isEnd = isEnd; |
||||
|
} |
||||
|
|
||||
|
private int fromLineNum; |
||||
|
private int toLineNum; |
||||
|
private String logContent; |
||||
|
private boolean isEnd; |
||||
|
|
||||
|
public int getFromLineNum() { |
||||
|
return fromLineNum; |
||||
|
} |
||||
|
|
||||
|
public void setFromLineNum(int fromLineNum) { |
||||
|
this.fromLineNum = fromLineNum; |
||||
|
} |
||||
|
|
||||
|
public int getToLineNum() { |
||||
|
return toLineNum; |
||||
|
} |
||||
|
|
||||
|
public void setToLineNum(int toLineNum) { |
||||
|
this.toLineNum = toLineNum; |
||||
|
} |
||||
|
|
||||
|
public String getLogContent() { |
||||
|
return logContent; |
||||
|
} |
||||
|
|
||||
|
public void setLogContent(String logContent) { |
||||
|
this.logContent = logContent; |
||||
|
} |
||||
|
|
||||
|
public boolean isEnd() { |
||||
|
return isEnd; |
||||
|
} |
||||
|
|
||||
|
public void setEnd(boolean end) { |
||||
|
isEnd = end; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,13 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
/** |
||||
|
* Created by xuxueli on 17/5/10. |
||||
|
*/ |
||||
|
public class RegistryConfig { |
||||
|
|
||||
|
public static final int BEAT_TIMEOUT = 30; |
||||
|
public static final int DEAD_TIMEOUT = BEAT_TIMEOUT * 3; |
||||
|
|
||||
|
public enum RegistType{ EXECUTOR, ADMIN } |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,51 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
import java.io.Serializable; |
||||
|
|
||||
|
public class RegistryParam implements Serializable { |
||||
|
private static final long serialVersionUID = 42L; |
||||
|
|
||||
|
private String registryGroup; |
||||
|
private String registryKey; |
||||
|
private String registryValue; |
||||
|
|
||||
|
public RegistryParam(){} |
||||
|
public RegistryParam(String registryGroup, String registryKey, String registryValue) { |
||||
|
this.registryGroup = registryGroup; |
||||
|
this.registryKey = registryKey; |
||||
|
this.registryValue = registryValue; |
||||
|
} |
||||
|
|
||||
|
public String getRegistryGroup() { |
||||
|
return registryGroup; |
||||
|
} |
||||
|
|
||||
|
public void setRegistryGroup(String registryGroup) { |
||||
|
this.registryGroup = registryGroup; |
||||
|
} |
||||
|
|
||||
|
public String getRegistryKey() { |
||||
|
return registryKey; |
||||
|
} |
||||
|
|
||||
|
public void setRegistryKey(String registryKey) { |
||||
|
this.registryKey = registryKey; |
||||
|
} |
||||
|
|
||||
|
public String getRegistryValue() { |
||||
|
return registryValue; |
||||
|
} |
||||
|
|
||||
|
public void setRegistryValue(String registryValue) { |
||||
|
this.registryValue = registryValue; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public String toString() { |
||||
|
return "RegistryParam{" + |
||||
|
"registryGroup='" + registryGroup + '\'' + |
||||
|
", registryKey='" + registryKey + '\'' + |
||||
|
", registryValue='" + registryValue + '\'' + |
||||
|
'}'; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,57 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
import java.io.Serializable; |
||||
|
|
||||
|
/** |
||||
|
* common return |
||||
|
* @author xuxueli 2015-12-4 16:32:31 |
||||
|
* @param <T> |
||||
|
*/ |
||||
|
public class ReturnT<T> implements Serializable { |
||||
|
public static final long serialVersionUID = 42L; |
||||
|
|
||||
|
public static final int SUCCESS_CODE = 200; |
||||
|
public static final int FAIL_CODE = 500; |
||||
|
|
||||
|
public static final ReturnT<String> SUCCESS = new ReturnT<String>(null); |
||||
|
public static final ReturnT<String> FAIL = new ReturnT<String>(FAIL_CODE, null); |
||||
|
|
||||
|
private int code; |
||||
|
private String msg; |
||||
|
private T content; |
||||
|
|
||||
|
public ReturnT(){} |
||||
|
public ReturnT(int code, String msg) { |
||||
|
this.code = code; |
||||
|
this.msg = msg; |
||||
|
} |
||||
|
public ReturnT(T content) { |
||||
|
this.code = SUCCESS_CODE; |
||||
|
this.content = content; |
||||
|
} |
||||
|
|
||||
|
public int getCode() { |
||||
|
return code; |
||||
|
} |
||||
|
public void setCode(int code) { |
||||
|
this.code = code; |
||||
|
} |
||||
|
public String getMsg() { |
||||
|
return msg; |
||||
|
} |
||||
|
public void setMsg(String msg) { |
||||
|
this.msg = msg; |
||||
|
} |
||||
|
public T getContent() { |
||||
|
return content; |
||||
|
} |
||||
|
public void setContent(T content) { |
||||
|
this.content = content; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public String toString() { |
||||
|
return "ReturnT [code=" + code + ", msg=" + msg + ", content=" + content + "]"; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,139 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
import java.io.Serializable; |
||||
|
|
||||
|
public class TriggerParam implements Serializable{ |
||||
|
private static final long serialVersionUID = 42L; |
||||
|
private int jobId; |
||||
|
private String executorHandler; |
||||
|
private String executorParams; |
||||
|
private String executorBlockStrategy; |
||||
|
private int executorTimeout; |
||||
|
|
||||
|
private long logId; |
||||
|
private long logDateTime; |
||||
|
|
||||
|
private String glueType; |
||||
|
private String glueSource; |
||||
|
private long glueUpdatetime; |
||||
|
|
||||
|
private int broadcastIndex; |
||||
|
private int broadcastTotal; |
||||
|
|
||||
|
|
||||
|
public int getJobId() { |
||||
|
return jobId; |
||||
|
} |
||||
|
|
||||
|
public void setJobId(int jobId) { |
||||
|
this.jobId = jobId; |
||||
|
} |
||||
|
|
||||
|
public String getExecutorHandler() { |
||||
|
return executorHandler; |
||||
|
} |
||||
|
|
||||
|
public void setExecutorHandler(String executorHandler) { |
||||
|
this.executorHandler = executorHandler; |
||||
|
} |
||||
|
|
||||
|
public String getExecutorParams() { |
||||
|
return executorParams; |
||||
|
} |
||||
|
|
||||
|
public void setExecutorParams(String executorParams) { |
||||
|
this.executorParams = executorParams; |
||||
|
} |
||||
|
|
||||
|
public String getExecutorBlockStrategy() { |
||||
|
return executorBlockStrategy; |
||||
|
} |
||||
|
|
||||
|
public void setExecutorBlockStrategy(String executorBlockStrategy) { |
||||
|
this.executorBlockStrategy = executorBlockStrategy; |
||||
|
} |
||||
|
|
||||
|
public int getExecutorTimeout() { |
||||
|
return executorTimeout; |
||||
|
} |
||||
|
|
||||
|
public void setExecutorTimeout(int executorTimeout) { |
||||
|
this.executorTimeout = executorTimeout; |
||||
|
} |
||||
|
|
||||
|
public long getLogId() { |
||||
|
return logId; |
||||
|
} |
||||
|
|
||||
|
public void setLogId(long logId) { |
||||
|
this.logId = logId; |
||||
|
} |
||||
|
|
||||
|
public long getLogDateTime() { |
||||
|
return logDateTime; |
||||
|
} |
||||
|
|
||||
|
public void setLogDateTime(long logDateTime) { |
||||
|
this.logDateTime = logDateTime; |
||||
|
} |
||||
|
|
||||
|
public String getGlueType() { |
||||
|
return glueType; |
||||
|
} |
||||
|
|
||||
|
public void setGlueType(String glueType) { |
||||
|
this.glueType = glueType; |
||||
|
} |
||||
|
|
||||
|
public String getGlueSource() { |
||||
|
return glueSource; |
||||
|
} |
||||
|
|
||||
|
public void setGlueSource(String glueSource) { |
||||
|
this.glueSource = glueSource; |
||||
|
} |
||||
|
|
||||
|
public long getGlueUpdatetime() { |
||||
|
return glueUpdatetime; |
||||
|
} |
||||
|
|
||||
|
public void setGlueUpdatetime(long glueUpdatetime) { |
||||
|
this.glueUpdatetime = glueUpdatetime; |
||||
|
} |
||||
|
|
||||
|
public int getBroadcastIndex() { |
||||
|
return broadcastIndex; |
||||
|
} |
||||
|
|
||||
|
public void setBroadcastIndex(int broadcastIndex) { |
||||
|
this.broadcastIndex = broadcastIndex; |
||||
|
} |
||||
|
|
||||
|
public int getBroadcastTotal() { |
||||
|
return broadcastTotal; |
||||
|
} |
||||
|
|
||||
|
public void setBroadcastTotal(int broadcastTotal) { |
||||
|
this.broadcastTotal = broadcastTotal; |
||||
|
} |
||||
|
|
||||
|
|
||||
|
@Override |
||||
|
public String toString() { |
||||
|
return "TriggerParam{" + |
||||
|
"jobId=" + jobId + |
||||
|
", executorHandler='" + executorHandler + '\'' + |
||||
|
", executorParams='" + executorParams + '\'' + |
||||
|
", executorBlockStrategy='" + executorBlockStrategy + '\'' + |
||||
|
", executorTimeout=" + executorTimeout + |
||||
|
", logId=" + logId + |
||||
|
", logDateTime=" + logDateTime + |
||||
|
", glueType='" + glueType + '\'' + |
||||
|
", glueSource='" + glueSource + '\'' + |
||||
|
", glueUpdatetime=" + glueUpdatetime + |
||||
|
", broadcastIndex=" + broadcastIndex + |
||||
|
", broadcastTotal=" + broadcastTotal + |
||||
|
'}'; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,79 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
/** |
||||
|
* xxl-job context |
||||
|
* |
||||
|
* @author xuxueli 2020-05-21 |
||||
|
* [Dear hj] |
||||
|
*/ |
||||
|
public class XxlJobContext { |
||||
|
public static final int HANDLE_CODE_SUCCESS = 200; |
||||
|
public static final int HANDLE_CODE_FAIL = 500; |
||||
|
public static final int HANDLE_CODE_TIMEOUT = 502; |
||||
|
|
||||
|
private static InheritableThreadLocal<XxlJobContext> contextHolder = new InheritableThreadLocal<XxlJobContext>(); // support for child thread of job handler)
|
||||
|
|
||||
|
public static void setXxlJobContext(XxlJobContext xxlJobContext){ |
||||
|
contextHolder.set(xxlJobContext); |
||||
|
} |
||||
|
|
||||
|
public static XxlJobContext getXxlJobContext(){ |
||||
|
return contextHolder.get(); |
||||
|
} |
||||
|
|
||||
|
private final long jobId; |
||||
|
private final String jobParam; |
||||
|
private final String jobLogFileName; |
||||
|
private final int shardIndex; |
||||
|
private final int shardTotal; |
||||
|
private int handleCode; |
||||
|
private String handleMsg; |
||||
|
|
||||
|
|
||||
|
|
||||
|
|
||||
|
public XxlJobContext(long jobId, String jobParam, String jobLogFileName, int shardIndex, int shardTotal) { |
||||
|
this.jobId = jobId; |
||||
|
this.jobParam = jobParam; |
||||
|
this.jobLogFileName = jobLogFileName; |
||||
|
this.shardIndex = shardIndex; |
||||
|
this.shardTotal = shardTotal; |
||||
|
this.handleCode = HANDLE_CODE_SUCCESS; // default success
|
||||
|
} |
||||
|
|
||||
|
public long getJobId() { |
||||
|
return jobId; |
||||
|
} |
||||
|
|
||||
|
public String getJobParam() { |
||||
|
return jobParam; |
||||
|
} |
||||
|
|
||||
|
public String getJobLogFileName() { |
||||
|
return jobLogFileName; |
||||
|
} |
||||
|
|
||||
|
public int getShardIndex() { |
||||
|
return shardIndex; |
||||
|
} |
||||
|
|
||||
|
public int getShardTotal() { |
||||
|
return shardTotal; |
||||
|
} |
||||
|
|
||||
|
public void setHandleCode(int handleCode) { |
||||
|
this.handleCode = handleCode; |
||||
|
} |
||||
|
|
||||
|
public int getHandleCode() { |
||||
|
return handleCode; |
||||
|
} |
||||
|
|
||||
|
public void setHandleMsg(String handleMsg) { |
||||
|
this.handleMsg = handleMsg; |
||||
|
} |
||||
|
|
||||
|
public String getHandleMsg() { |
||||
|
return handleMsg; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,96 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
import io.sc.platform.job.executor.thread.JobLogFileCleanThread; |
||||
|
import io.sc.platform.job.executor.thread.TriggerCallbackThread; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.List; |
||||
|
|
||||
|
/** |
||||
|
* Created by xuxueli on 2016/3/2 21:14. |
||||
|
*/ |
||||
|
public class XxlJobExecutor { |
||||
|
private static final Logger logger = LoggerFactory.getLogger(XxlJobExecutor.class); |
||||
|
|
||||
|
// ---------------------- param ----------------------
|
||||
|
private String adminAddresses; |
||||
|
private String accessToken; |
||||
|
private String appname; |
||||
|
private String address; |
||||
|
private String ip; |
||||
|
private int port; |
||||
|
private String logPath; |
||||
|
private int logRetentionDays; |
||||
|
|
||||
|
public void setAdminAddresses(String adminAddresses) { |
||||
|
this.adminAddresses = adminAddresses; |
||||
|
} |
||||
|
public void setAccessToken(String accessToken) { |
||||
|
this.accessToken = accessToken; |
||||
|
} |
||||
|
public void setAppname(String appname) { |
||||
|
this.appname = appname; |
||||
|
} |
||||
|
public void setAddress(String address) { |
||||
|
this.address = address; |
||||
|
} |
||||
|
public void setIp(String ip) { |
||||
|
this.ip = ip; |
||||
|
} |
||||
|
public void setPort(int port) { |
||||
|
this.port = port; |
||||
|
} |
||||
|
public void setLogPath(String logPath) { |
||||
|
this.logPath = logPath; |
||||
|
} |
||||
|
public void setLogRetentionDays(int logRetentionDays) { |
||||
|
this.logRetentionDays = logRetentionDays; |
||||
|
} |
||||
|
|
||||
|
|
||||
|
// ---------------------- start + stop ----------------------
|
||||
|
public void start() throws Exception { |
||||
|
|
||||
|
// init logpath
|
||||
|
XxlJobFileAppender.initLogPath(logPath); |
||||
|
|
||||
|
// init invoker, admin-client
|
||||
|
initAdminBizList(adminAddresses, accessToken); |
||||
|
|
||||
|
|
||||
|
// init JobLogFileCleanThread
|
||||
|
JobLogFileCleanThread.getInstance().start(logRetentionDays); |
||||
|
|
||||
|
// init TriggerCallbackThread
|
||||
|
TriggerCallbackThread.getInstance().start(); |
||||
|
|
||||
|
// init executor-server
|
||||
|
} |
||||
|
|
||||
|
|
||||
|
|
||||
|
// ---------------------- admin-client (rpc invoker) ----------------------
|
||||
|
private static List<AdminBiz> adminBizList; |
||||
|
private void initAdminBizList(String adminAddresses, String accessToken) throws Exception { |
||||
|
if (adminAddresses!=null && adminAddresses.trim().length()>0) { |
||||
|
for (String address: adminAddresses.trim().split(",")) { |
||||
|
if (address!=null && address.trim().length()>0) { |
||||
|
|
||||
|
AdminBiz adminBiz = new AdminBizClient(address.trim(), accessToken); |
||||
|
|
||||
|
if (adminBizList == null) { |
||||
|
adminBizList = new ArrayList<AdminBiz>(); |
||||
|
} |
||||
|
adminBizList.add(adminBiz); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public static List<AdminBiz> getAdminBizList(){ |
||||
|
return adminBizList; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,219 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
import java.io.*; |
||||
|
import java.text.SimpleDateFormat; |
||||
|
import java.util.Date; |
||||
|
|
||||
|
/** |
||||
|
* store trigger log in each log-file |
||||
|
* @author xuxueli 2016-3-12 19:25:12 |
||||
|
*/ |
||||
|
public class XxlJobFileAppender { |
||||
|
private static Logger logger = LoggerFactory.getLogger(XxlJobFileAppender.class); |
||||
|
|
||||
|
/** |
||||
|
* log base path |
||||
|
* |
||||
|
* strut like: |
||||
|
* ---/ |
||||
|
* ---/gluesource/ |
||||
|
* ---/gluesource/10_1514171108000.js |
||||
|
* ---/gluesource/10_1514171108000.js |
||||
|
* ---/2017-12-25/ |
||||
|
* ---/2017-12-25/639.log |
||||
|
* ---/2017-12-25/821.log |
||||
|
* |
||||
|
*/ |
||||
|
private static String logBasePath = "/data/applogs/xxl-job/jobhandler"; |
||||
|
private static String glueSrcPath = logBasePath.concat("/gluesource"); |
||||
|
public static void initLogPath(String logPath){ |
||||
|
// init
|
||||
|
if (logPath!=null && logPath.trim().length()>0) { |
||||
|
logBasePath = logPath; |
||||
|
} |
||||
|
// mk base dir
|
||||
|
File logPathDir = new File(logBasePath); |
||||
|
if (!logPathDir.exists()) { |
||||
|
logPathDir.mkdirs(); |
||||
|
} |
||||
|
logBasePath = logPathDir.getPath(); |
||||
|
|
||||
|
// mk glue dir
|
||||
|
File glueBaseDir = new File(logPathDir, "gluesource"); |
||||
|
if (!glueBaseDir.exists()) { |
||||
|
glueBaseDir.mkdirs(); |
||||
|
} |
||||
|
glueSrcPath = glueBaseDir.getPath(); |
||||
|
} |
||||
|
public static String getLogPath() { |
||||
|
return logBasePath; |
||||
|
} |
||||
|
public static String getGlueSrcPath() { |
||||
|
return glueSrcPath; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* log filename, like "logPath/yyyy-MM-dd/9999.log" |
||||
|
* |
||||
|
* @param triggerDate |
||||
|
* @param logId |
||||
|
* @return |
||||
|
*/ |
||||
|
public static String makeLogFileName(Date triggerDate, long logId) { |
||||
|
|
||||
|
// filePath/yyyy-MM-dd
|
||||
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); // avoid concurrent problem, can not be static
|
||||
|
File logFilePath = new File(getLogPath(), sdf.format(triggerDate)); |
||||
|
if (!logFilePath.exists()) { |
||||
|
logFilePath.mkdir(); |
||||
|
} |
||||
|
|
||||
|
// filePath/yyyy-MM-dd/9999.log
|
||||
|
String logFileName = logFilePath.getPath() |
||||
|
.concat(File.separator) |
||||
|
.concat(String.valueOf(logId)) |
||||
|
.concat(".log"); |
||||
|
return logFileName; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* append log |
||||
|
* |
||||
|
* @param logFileName |
||||
|
* @param appendLog |
||||
|
*/ |
||||
|
public static void appendLog(String logFileName, String appendLog) { |
||||
|
|
||||
|
// log file
|
||||
|
if (logFileName==null || logFileName.trim().length()==0) { |
||||
|
return; |
||||
|
} |
||||
|
File logFile = new File(logFileName); |
||||
|
|
||||
|
if (!logFile.exists()) { |
||||
|
try { |
||||
|
logFile.createNewFile(); |
||||
|
} catch (IOException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
return; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// log
|
||||
|
if (appendLog == null) { |
||||
|
appendLog = ""; |
||||
|
} |
||||
|
appendLog += "\r\n"; |
||||
|
|
||||
|
// append file content
|
||||
|
FileOutputStream fos = null; |
||||
|
try { |
||||
|
fos = new FileOutputStream(logFile, true); |
||||
|
fos.write(appendLog.getBytes("utf-8")); |
||||
|
fos.flush(); |
||||
|
} catch (Exception e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} finally { |
||||
|
if (fos != null) { |
||||
|
try { |
||||
|
fos.close(); |
||||
|
} catch (IOException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* support read log-file |
||||
|
* |
||||
|
* @param logFileName |
||||
|
* @return log content |
||||
|
*/ |
||||
|
public static LogResult readLog(String logFileName, int fromLineNum){ |
||||
|
|
||||
|
// valid log file
|
||||
|
if (logFileName==null || logFileName.trim().length()==0) { |
||||
|
return new LogResult(fromLineNum, 0, "readLog fail, logFile not found", true); |
||||
|
} |
||||
|
File logFile = new File(logFileName); |
||||
|
|
||||
|
if (!logFile.exists()) { |
||||
|
return new LogResult(fromLineNum, 0, "readLog fail, logFile not exists", true); |
||||
|
} |
||||
|
|
||||
|
// read file
|
||||
|
StringBuffer logContentBuffer = new StringBuffer(); |
||||
|
int toLineNum = 0; |
||||
|
LineNumberReader reader = null; |
||||
|
try { |
||||
|
//reader = new LineNumberReader(new FileReader(logFile));
|
||||
|
reader = new LineNumberReader(new InputStreamReader(new FileInputStream(logFile), "utf-8")); |
||||
|
String line = null; |
||||
|
|
||||
|
while ((line = reader.readLine())!=null) { |
||||
|
toLineNum = reader.getLineNumber(); // [from, to], start as 1
|
||||
|
if (toLineNum >= fromLineNum) { |
||||
|
logContentBuffer.append(line).append("\n"); |
||||
|
} |
||||
|
} |
||||
|
} catch (IOException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} finally { |
||||
|
if (reader != null) { |
||||
|
try { |
||||
|
reader.close(); |
||||
|
} catch (IOException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// result
|
||||
|
LogResult logResult = new LogResult(fromLineNum, toLineNum, logContentBuffer.toString(), false); |
||||
|
return logResult; |
||||
|
|
||||
|
/* |
||||
|
// it will return the number of characters actually skipped
|
||||
|
reader.skip(Long.MAX_VALUE); |
||||
|
int maxLineNum = reader.getLineNumber(); |
||||
|
maxLineNum++; // 最大行号
|
||||
|
*/ |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* read log data |
||||
|
* @param logFile |
||||
|
* @return log line content |
||||
|
*/ |
||||
|
public static String readLines(File logFile){ |
||||
|
BufferedReader reader = null; |
||||
|
try { |
||||
|
reader = new BufferedReader(new InputStreamReader(new FileInputStream(logFile), "utf-8")); |
||||
|
if (reader != null) { |
||||
|
StringBuilder sb = new StringBuilder(); |
||||
|
String line = null; |
||||
|
while ((line = reader.readLine()) != null) { |
||||
|
sb.append(line).append("\n"); |
||||
|
} |
||||
|
return sb.toString(); |
||||
|
} |
||||
|
} catch (IOException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} finally { |
||||
|
if (reader != null) { |
||||
|
try { |
||||
|
reader.close(); |
||||
|
} catch (IOException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,202 @@ |
|||||
|
package io.sc.platform.job.executor; |
||||
|
|
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
import org.slf4j.helpers.FormattingTuple; |
||||
|
import org.slf4j.helpers.MessageFormatter; |
||||
|
|
||||
|
import java.io.PrintWriter; |
||||
|
import java.io.StringWriter; |
||||
|
import java.util.Date; |
||||
|
import io.sc.platform.job.executor.util.DateUtil; |
||||
|
|
||||
|
/** |
||||
|
* helper for xxl-job |
||||
|
* |
||||
|
* @author xuxueli 2020-11-05 |
||||
|
*/ |
||||
|
public class XxlJobHelper { |
||||
|
private static Logger logger = LoggerFactory.getLogger("xxl-job logger"); |
||||
|
|
||||
|
public static long getJobId() { |
||||
|
XxlJobContext xxlJobContext = XxlJobContext.getXxlJobContext(); |
||||
|
if (xxlJobContext == null) { |
||||
|
return -1; |
||||
|
} |
||||
|
return xxlJobContext.getJobId(); |
||||
|
} |
||||
|
|
||||
|
public static String getJobParam() { |
||||
|
XxlJobContext xxlJobContext = XxlJobContext.getXxlJobContext(); |
||||
|
if (xxlJobContext == null) { |
||||
|
return null; |
||||
|
} |
||||
|
return xxlJobContext.getJobParam(); |
||||
|
} |
||||
|
|
||||
|
public static String getJobLogFileName() { |
||||
|
XxlJobContext xxlJobContext = XxlJobContext.getXxlJobContext(); |
||||
|
if (xxlJobContext == null) { |
||||
|
return null; |
||||
|
} |
||||
|
return xxlJobContext.getJobLogFileName(); |
||||
|
} |
||||
|
|
||||
|
public static int getShardIndex() { |
||||
|
XxlJobContext xxlJobContext = XxlJobContext.getXxlJobContext(); |
||||
|
if (xxlJobContext == null) { |
||||
|
return -1; |
||||
|
} |
||||
|
return xxlJobContext.getShardIndex(); |
||||
|
} |
||||
|
|
||||
|
public static int getShardTotal() { |
||||
|
XxlJobContext xxlJobContext = XxlJobContext.getXxlJobContext(); |
||||
|
if (xxlJobContext == null) { |
||||
|
return -1; |
||||
|
} |
||||
|
return xxlJobContext.getShardTotal(); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* append log with pattern |
||||
|
* |
||||
|
* @param appendLogPattern like "aaa {} bbb {} ccc" |
||||
|
* @param appendLogArguments like "111, true" |
||||
|
*/ |
||||
|
public static boolean log(String appendLogPattern, Object ... appendLogArguments) { |
||||
|
FormattingTuple ft = MessageFormatter.arrayFormat(appendLogPattern, appendLogArguments); |
||||
|
String appendLog = ft.getMessage(); |
||||
|
StackTraceElement callInfo = new Throwable().getStackTrace()[1]; |
||||
|
return logDetail(callInfo, appendLog); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* append exception stack |
||||
|
* |
||||
|
* @param e |
||||
|
*/ |
||||
|
public static boolean log(Throwable e) { |
||||
|
StringWriter stringWriter = new StringWriter(); |
||||
|
e.printStackTrace(new PrintWriter(stringWriter)); |
||||
|
String appendLog = stringWriter.toString(); |
||||
|
StackTraceElement callInfo = new Throwable().getStackTrace()[1]; |
||||
|
return logDetail(callInfo, appendLog); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* append log |
||||
|
* |
||||
|
* @param callInfo |
||||
|
* @param appendLog |
||||
|
*/ |
||||
|
private static boolean logDetail(StackTraceElement callInfo, String appendLog) { |
||||
|
XxlJobContext xxlJobContext = XxlJobContext.getXxlJobContext(); |
||||
|
if (xxlJobContext == null) { |
||||
|
return false; |
||||
|
} |
||||
|
|
||||
|
StringBuffer stringBuffer = new StringBuffer(); |
||||
|
stringBuffer.append(DateUtil.formatDateTime(new Date())).append(" ") |
||||
|
.append("["+ callInfo.getClassName() + "#" + callInfo.getMethodName() +"]").append("-") |
||||
|
.append("["+ callInfo.getLineNumber() +"]").append("-") |
||||
|
.append("["+ Thread.currentThread().getName() +"]").append(" ") |
||||
|
.append(appendLog!=null?appendLog:""); |
||||
|
String formatAppendLog = stringBuffer.toString(); |
||||
|
|
||||
|
// appendlog
|
||||
|
String logFileName = xxlJobContext.getJobLogFileName(); |
||||
|
|
||||
|
if (logFileName!=null && logFileName.trim().length()>0) { |
||||
|
XxlJobFileAppender.appendLog(logFileName, formatAppendLog); |
||||
|
return true; |
||||
|
} else { |
||||
|
logger.info(">>>>>>>>>>> {}", formatAppendLog); |
||||
|
return false; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// ---------------------- tool for handleResult ----------------------
|
||||
|
|
||||
|
/** |
||||
|
* handle success |
||||
|
* |
||||
|
* @return |
||||
|
*/ |
||||
|
public static boolean handleSuccess(){ |
||||
|
return handleResult(XxlJobContext.HANDLE_CODE_SUCCESS, null); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* handle success with log msg |
||||
|
* |
||||
|
* @param handleMsg |
||||
|
* @return |
||||
|
*/ |
||||
|
public static boolean handleSuccess(String handleMsg) { |
||||
|
return handleResult(XxlJobContext.HANDLE_CODE_SUCCESS, handleMsg); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* handle fail |
||||
|
* |
||||
|
* @return |
||||
|
*/ |
||||
|
public static boolean handleFail(){ |
||||
|
return handleResult(XxlJobContext.HANDLE_CODE_FAIL, null); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* handle fail with log msg |
||||
|
* |
||||
|
* @param handleMsg |
||||
|
* @return |
||||
|
*/ |
||||
|
public static boolean handleFail(String handleMsg) { |
||||
|
return handleResult(XxlJobContext.HANDLE_CODE_FAIL, handleMsg); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* handle timeout |
||||
|
* |
||||
|
* @return |
||||
|
*/ |
||||
|
public static boolean handleTimeout(){ |
||||
|
return handleResult(XxlJobContext.HANDLE_CODE_TIMEOUT, null); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* handle timeout with log msg |
||||
|
* |
||||
|
* @param handleMsg |
||||
|
* @return |
||||
|
*/ |
||||
|
public static boolean handleTimeout(String handleMsg){ |
||||
|
return handleResult(XxlJobContext.HANDLE_CODE_TIMEOUT, handleMsg); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* @param handleCode |
||||
|
* |
||||
|
* 200 : success |
||||
|
* 500 : fail |
||||
|
* 502 : timeout |
||||
|
* |
||||
|
* @param handleMsg |
||||
|
* @return |
||||
|
*/ |
||||
|
public static boolean handleResult(int handleCode, String handleMsg) { |
||||
|
XxlJobContext xxlJobContext = XxlJobContext.getXxlJobContext(); |
||||
|
if (xxlJobContext == null) { |
||||
|
return false; |
||||
|
} |
||||
|
|
||||
|
xxlJobContext.setHandleCode(handleCode); |
||||
|
if (handleMsg != null) { |
||||
|
xxlJobContext.setHandleMsg(handleMsg); |
||||
|
} |
||||
|
return true; |
||||
|
} |
||||
|
|
||||
|
|
||||
|
} |
||||
@ -0,0 +1,7 @@ |
|||||
|
package io.sc.platform.job.executor.client; |
||||
|
|
||||
|
import io.sc.platform.job.core.TaskLog; |
||||
|
|
||||
|
public interface ExecutorCallbackClient { |
||||
|
public void callback(String url, TaskLog taskLog); |
||||
|
} |
||||
@ -0,0 +1,8 @@ |
|||||
|
package io.sc.platform.job.executor.client; |
||||
|
|
||||
|
import io.sc.platform.job.core.ExecutorRegistry; |
||||
|
|
||||
|
public interface ExecutorRegistryClient { |
||||
|
public boolean registry(String url, ExecutorRegistry executorRegistry); |
||||
|
public boolean unRegistry(String url, ExecutorRegistry executorRegistry); |
||||
|
} |
||||
@ -0,0 +1,27 @@ |
|||||
|
package io.sc.platform.job.executor.client.impl; |
||||
|
|
||||
|
import io.sc.platform.core.response.ResponseWrapper; |
||||
|
import io.sc.platform.core.util.UrlUtil; |
||||
|
import io.sc.platform.job.core.TaskLog; |
||||
|
import io.sc.platform.job.executor.client.ExecutorCallbackClient; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.springframework.web.client.RestClientException; |
||||
|
import org.springframework.web.client.RestTemplate; |
||||
|
|
||||
|
@Service |
||||
|
public class ExecutorCallbackClientImpl implements ExecutorCallbackClient { |
||||
|
private static final Logger logger = LoggerFactory.getLogger(ExecutorCallbackClientImpl.class); |
||||
|
@Autowired private RestTemplate restTemplate; |
||||
|
|
||||
|
@Override |
||||
|
public void callback(String url, TaskLog taskLog) { |
||||
|
try { |
||||
|
ResponseWrapper response = restTemplate.postForObject(UrlUtil.concatUrl(url,"callback"), taskLog, ResponseWrapper.class); |
||||
|
}catch (RestClientException e){ |
||||
|
logger.error("",e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,40 @@ |
|||||
|
package io.sc.platform.job.executor.client.impl; |
||||
|
|
||||
|
import io.sc.platform.core.response.ResponseWrapper; |
||||
|
import io.sc.platform.core.util.UrlUtil; |
||||
|
import io.sc.platform.job.core.ExecutorRegistry; |
||||
|
import io.sc.platform.job.executor.client.ExecutorRegistryClient; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.springframework.web.client.RestClientException; |
||||
|
import org.springframework.web.client.RestTemplate; |
||||
|
|
||||
|
@Service |
||||
|
public class ExecutorRegistryClientImpl implements ExecutorRegistryClient { |
||||
|
private static final Logger logger = LoggerFactory.getLogger(ExecutorRegistryClientImpl.class); |
||||
|
@Autowired private RestTemplate restTemplate; |
||||
|
|
||||
|
@Override |
||||
|
public boolean registry(String url, ExecutorRegistry executorRegistry) { |
||||
|
try { |
||||
|
ResponseWrapper response = restTemplate.postForObject(UrlUtil.concatUrl(url,"registry"), executorRegistry, ResponseWrapper.class); |
||||
|
return response.isSuccess(); |
||||
|
}catch (RestClientException e){ |
||||
|
logger.error("",e); |
||||
|
return false; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean unRegistry(String url, ExecutorRegistry executorRegistry) { |
||||
|
try { |
||||
|
ResponseWrapper response = restTemplate.postForObject(UrlUtil.concatUrl(url,"unRegistry"), executorRegistry, ResponseWrapper.class); |
||||
|
return response.isSuccess(); |
||||
|
}catch (RestClientException e){ |
||||
|
logger.error("",e); |
||||
|
return false; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,58 @@ |
|||||
|
package io.sc.platform.job.executor.component; |
||||
|
|
||||
|
import io.sc.platform.core.service.RuntimeService; |
||||
|
import io.sc.platform.job.executor.thread.ExecutorRegistryThread; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.boot.context.event.ApplicationReadyEvent; |
||||
|
import org.springframework.cloud.context.environment.EnvironmentChangeEvent; |
||||
|
import org.springframework.context.ApplicationContext; |
||||
|
import org.springframework.context.ApplicationListener; |
||||
|
import org.springframework.context.event.ContextClosedEvent; |
||||
|
import org.springframework.context.event.EventListener; |
||||
|
import org.springframework.stereotype.Component; |
||||
|
|
||||
|
import java.util.Set; |
||||
|
|
||||
|
@Component |
||||
|
public class ExecutorInitializer implements ApplicationListener<ApplicationReadyEvent> { |
||||
|
@Autowired private ApplicationContext applicationContext; |
||||
|
@Autowired private RuntimeService runtimeService; |
||||
|
|
||||
|
private ExecutorRegistryThread registryThread; |
||||
|
|
||||
|
@Override |
||||
|
public void onApplicationEvent(ApplicationReadyEvent event) { |
||||
|
if(runtimeService.isReady()) { |
||||
|
registryThread = new ExecutorRegistryThread(applicationContext); |
||||
|
registryThread.start(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@EventListener |
||||
|
public void environmentChangeEventHandle(ContextClosedEvent event){ |
||||
|
if(runtimeService.isReady()) { |
||||
|
if (registryThread != null) { |
||||
|
registryThread.shutdown(); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@EventListener |
||||
|
public void environmentChangeEventHandle(EnvironmentChangeEvent event){ |
||||
|
if(runtimeService.isReady()) { |
||||
|
// 获取变换的属性
|
||||
|
Set<String> changedKeys = event.getKeys(); |
||||
|
for (String key : changedKeys) { |
||||
|
// 如果是数据源配置信息发生变化
|
||||
|
if (key.startsWith("job.manager.urls[")) { |
||||
|
if (registryThread != null) { |
||||
|
registryThread.shutdown(); |
||||
|
} |
||||
|
registryThread = new ExecutorRegistryThread(applicationContext); |
||||
|
registryThread.start(); |
||||
|
break; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,9 @@ |
|||||
|
package io.sc.platform.job.executor.configure; |
||||
|
|
||||
|
import org.springframework.boot.context.properties.EnableConfigurationProperties; |
||||
|
import org.springframework.context.annotation.Configuration; |
||||
|
|
||||
|
@Configuration |
||||
|
@EnableConfigurationProperties({ManagerProperties.class}) |
||||
|
public class ExecutorConfigurationAutoConfiguration { |
||||
|
} |
||||
@ -0,0 +1,19 @@ |
|||||
|
package io.sc.platform.job.executor.configure; |
||||
|
|
||||
|
import org.springframework.boot.context.properties.ConfigurationProperties; |
||||
|
|
||||
|
import java.util.HashSet; |
||||
|
import java.util.Set; |
||||
|
|
||||
|
@ConfigurationProperties("job.manager") |
||||
|
public class ManagerProperties { |
||||
|
private Set<String> urls =new HashSet<>(); |
||||
|
|
||||
|
public Set<String> getUrls() { |
||||
|
return urls; |
||||
|
} |
||||
|
|
||||
|
public void setUrls(Set<String> urls) { |
||||
|
this.urls = urls; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,38 @@ |
|||||
|
package io.sc.platform.job.executor.controller; |
||||
|
|
||||
|
import io.sc.platform.job.executor.service.ExecutorService; |
||||
|
import org.springframework.stereotype.Controller; |
||||
|
import org.springframework.web.bind.annotation.*; |
||||
|
|
||||
|
import javax.annotation.Resource; |
||||
|
|
||||
|
@RestController |
||||
|
@RequestMapping("/api/job/executor") |
||||
|
public class ExecutorWebController { |
||||
|
@Resource private ExecutorService service; |
||||
|
|
||||
|
@PostMapping("/beat") |
||||
|
public void beat(){ |
||||
|
service.beat(); |
||||
|
} |
||||
|
|
||||
|
@PostMapping("/idleBeat/{jobId}") |
||||
|
public boolean idleBeat(@PathVariable("jobId") String jobId){ |
||||
|
return service.idleBeat(jobId); |
||||
|
} |
||||
|
|
||||
|
@PostMapping("run") |
||||
|
public void run(){ |
||||
|
service.run(); |
||||
|
} |
||||
|
|
||||
|
@PostMapping("kill") |
||||
|
public void kill(){ |
||||
|
service.kill(); |
||||
|
} |
||||
|
|
||||
|
@PostMapping("log") |
||||
|
public void log(){ |
||||
|
service.log(); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,22 @@ |
|||||
|
package io.sc.platform.job.executor.manager; |
||||
|
|
||||
|
import io.sc.platform.job.executor.JobHandler; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||
|
import java.util.concurrent.ConcurrentMap; |
||||
|
|
||||
|
public class JobHandlerManager { |
||||
|
private static Logger logger = LoggerFactory.getLogger(JobHandlerManager.class); |
||||
|
private static ConcurrentMap<String, JobHandler> jobHandlerRepository = new ConcurrentHashMap<String, JobHandler>(); |
||||
|
|
||||
|
public static JobHandler loadJobHandler(String name){ |
||||
|
return jobHandlerRepository.get(name); |
||||
|
} |
||||
|
public static JobHandler registJobHandler(String name, JobHandler jobHandler){ |
||||
|
logger.info("register jobhandler success, name:{}, jobHandler:{}", name, jobHandler); |
||||
|
return jobHandlerRepository.put(name, jobHandler); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,43 @@ |
|||||
|
package io.sc.platform.job.executor.manager; |
||||
|
|
||||
|
import io.sc.platform.job.executor.JobHandler; |
||||
|
import io.sc.platform.job.executor.thread.JobThread; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||
|
import java.util.concurrent.ConcurrentMap; |
||||
|
|
||||
|
public class JobThreadManager { |
||||
|
private static Logger logger = LoggerFactory.getLogger(JobThreadManager.class); |
||||
|
private static ConcurrentMap<String, JobThread> jobThreadRepository = new ConcurrentHashMap<String, JobThread>(); |
||||
|
|
||||
|
public static JobThread registJobThread(String jobId, JobHandler handler, String removeOldReason){ |
||||
|
JobThread newJobThread = new JobThread(jobId, handler); |
||||
|
newJobThread.start(); |
||||
|
logger.info("regist JobThread success, jobId:{}, handler:{}", new Object[]{jobId, handler}); |
||||
|
|
||||
|
JobThread oldJobThread = jobThreadRepository.put(jobId, newJobThread); // putIfAbsent | oh my god, map's put method return the old value!!!
|
||||
|
if (oldJobThread != null) { |
||||
|
oldJobThread.toStop(removeOldReason); |
||||
|
oldJobThread.interrupt(); |
||||
|
} |
||||
|
|
||||
|
return newJobThread; |
||||
|
} |
||||
|
|
||||
|
public static JobThread removeJobThread(String jobId, String removeOldReason){ |
||||
|
JobThread oldJobThread = jobThreadRepository.remove(jobId); |
||||
|
if (oldJobThread != null) { |
||||
|
oldJobThread.toStop(removeOldReason); |
||||
|
oldJobThread.interrupt(); |
||||
|
|
||||
|
return oldJobThread; |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
public static JobThread loadJobThread(String jobId){ |
||||
|
return jobThreadRepository.get(jobId); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,16 @@ |
|||||
|
package io.sc.platform.job.executor.manager; |
||||
|
|
||||
|
import io.sc.platform.job.executor.configure.ManagerProperties; |
||||
|
|
||||
|
public class RegistryManager { |
||||
|
private ManagerProperties managerProperties; |
||||
|
|
||||
|
private static class RegistryManagerHolder{ |
||||
|
private static RegistryManager instance =new RegistryManager(); |
||||
|
} |
||||
|
|
||||
|
private RegistryManager(){} |
||||
|
public static RegistryManager getInstance(){ |
||||
|
return RegistryManagerHolder.instance; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,11 @@ |
|||||
|
package io.sc.platform.job.executor.service; |
||||
|
|
||||
|
import org.springframework.web.bind.annotation.PathVariable; |
||||
|
|
||||
|
public interface ExecutorService { |
||||
|
public void beat(); |
||||
|
public boolean idleBeat(String jobId); |
||||
|
public void run(); |
||||
|
public void kill(); |
||||
|
public void log(); |
||||
|
} |
||||
@ -0,0 +1,47 @@ |
|||||
|
package io.sc.platform.job.executor.service.impl; |
||||
|
|
||||
|
import io.sc.platform.job.executor.thread.JobThread; |
||||
|
import io.sc.platform.job.executor.manager.JobThreadManager; |
||||
|
import io.sc.platform.job.executor.service.ExecutorService; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
public class ExecutorServiceImpl implements ExecutorService { |
||||
|
private static Logger logger = LoggerFactory.getLogger(ExecutorServiceImpl.class); |
||||
|
|
||||
|
@Override |
||||
|
public void beat() { |
||||
|
|
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public boolean idleBeat(String jobId) { |
||||
|
// isRunningOrHasQueue
|
||||
|
boolean isRunningOrHasQueue = false; |
||||
|
JobThread jobThread = JobThreadManager.loadJobThread(jobId); |
||||
|
if (jobThread != null && jobThread.isRunningOrHasQueue()) { |
||||
|
isRunningOrHasQueue = true; |
||||
|
} |
||||
|
|
||||
|
if (isRunningOrHasQueue) { |
||||
|
logger.warn("job thread is running or has trigger queue."); |
||||
|
return false; |
||||
|
} |
||||
|
return true; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void run() { |
||||
|
|
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void kill() { |
||||
|
|
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void log() { |
||||
|
|
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,81 @@ |
|||||
|
package io.sc.platform.job.executor.thread; |
||||
|
|
||||
|
import io.sc.platform.core.Environment; |
||||
|
import io.sc.platform.core.util.IpUtil; |
||||
|
import io.sc.platform.job.core.ExecutorRegistry; |
||||
|
import io.sc.platform.job.executor.client.ExecutorRegistryClient; |
||||
|
import io.sc.platform.job.executor.configure.ManagerProperties; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
import org.springframework.context.ApplicationContext; |
||||
|
|
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
|
||||
|
public class ExecutorRegistryThread extends Thread { |
||||
|
private static final Logger logger = LoggerFactory.getLogger(ExecutorRegistryThread.class); |
||||
|
private static final int BEAT_TIMEOUT =30; |
||||
|
|
||||
|
private ManagerProperties managerProperties; |
||||
|
private ExecutorRegistryClient executorRegistryClient; |
||||
|
private String appName; |
||||
|
private String executorServiceUrl; |
||||
|
private volatile boolean running = true; |
||||
|
|
||||
|
public ExecutorRegistryThread(ApplicationContext applicationContext){ |
||||
|
this.managerProperties =applicationContext.getBean(ManagerProperties.class); |
||||
|
this.executorRegistryClient =applicationContext.getBean(ExecutorRegistryClient.class); |
||||
|
this.appName =Environment.getInstance().getApplicationName(); |
||||
|
this.executorServiceUrl =buildExecutorServiceUrl(applicationContext); |
||||
|
} |
||||
|
|
||||
|
public void shutdown(){ |
||||
|
this.running =false; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void run() { |
||||
|
if(managerProperties==null || managerProperties.getUrls()==null || managerProperties.getUrls().isEmpty()){ |
||||
|
StringBuilder sb =new StringBuilder("job.manager.urls property NOT set in application.properties, e.x.").append("\n"); |
||||
|
sb.append("job.manager.urls[0]=http://localhost:8080/api/job/manager/executor").append("\n"); |
||||
|
sb.append("job.manager.urls[1]=http://192.168.1.100:8080/api/job/manager/executor").append("\n"); |
||||
|
logger.error(sb.toString()); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
//固定频率注册
|
||||
|
while(running){ |
||||
|
ExecutorRegistry executorRegistry = new ExecutorRegistry(appName, executorServiceUrl); |
||||
|
for(String url : managerProperties.getUrls()){ |
||||
|
//只要一个注册成功即可
|
||||
|
if(executorRegistryClient.registry(url,executorRegistry)){ |
||||
|
logger.debug("registry success, {}", new Object[]{executorRegistry}); |
||||
|
break; |
||||
|
} |
||||
|
} |
||||
|
try { |
||||
|
if (running) { TimeUnit.SECONDS.sleep(BEAT_TIMEOUT); } |
||||
|
} catch (InterruptedException e) { |
||||
|
if (running) { logger.warn("executor registry thread interrupted, error msg:{}", e.getMessage()); } |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
//停止或打断后反注册
|
||||
|
ExecutorRegistry executorRegistry = new ExecutorRegistry(appName, executorServiceUrl); |
||||
|
for(String url : managerProperties.getUrls()){ |
||||
|
//只要一个反注册成功即可
|
||||
|
if(executorRegistryClient.unRegistry(url,executorRegistry)){ |
||||
|
logger.info("unregistry success, {}", new Object[]{executorRegistry}); |
||||
|
break; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private String buildExecutorServiceUrl(ApplicationContext applicationContext){ |
||||
|
StringBuilder sb =new StringBuilder(); |
||||
|
sb.append(applicationContext.getEnvironment().getProperty("server.ssl.enabled",Boolean.class,false)?"https":"http"); |
||||
|
sb.append("://").append(IpUtil.getLocalIp()); |
||||
|
sb.append(":").append(applicationContext.getEnvironment().getProperty("server.port",String.class,"8080")); |
||||
|
sb.append(applicationContext.getEnvironment().getProperty("server.servlet.context-path",String.class,"/")); |
||||
|
return sb.toString(); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,124 @@ |
|||||
|
package io.sc.platform.job.executor.thread; |
||||
|
|
||||
|
import io.sc.platform.job.executor.XxlJobFileAppender; |
||||
|
import io.sc.platform.job.executor.util.FileUtil; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.text.ParseException; |
||||
|
import java.text.SimpleDateFormat; |
||||
|
import java.util.Calendar; |
||||
|
import java.util.Date; |
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
|
||||
|
/** |
||||
|
* job file clean thread |
||||
|
* |
||||
|
* @author xuxueli 2017-12-29 16:23:43 |
||||
|
*/ |
||||
|
public class JobLogFileCleanThread { |
||||
|
private static Logger logger = LoggerFactory.getLogger(JobLogFileCleanThread.class); |
||||
|
|
||||
|
private static JobLogFileCleanThread instance = new JobLogFileCleanThread(); |
||||
|
public static JobLogFileCleanThread getInstance(){ |
||||
|
return instance; |
||||
|
} |
||||
|
|
||||
|
private Thread localThread; |
||||
|
private volatile boolean toStop = false; |
||||
|
public void start(final long logRetentionDays){ |
||||
|
|
||||
|
// limit min value
|
||||
|
if (logRetentionDays < 3 ) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
localThread = new Thread(new Runnable() { |
||||
|
@Override |
||||
|
public void run() { |
||||
|
while (!toStop) { |
||||
|
try { |
||||
|
// clean log dir, over logRetentionDays
|
||||
|
File[] childDirs = new File(XxlJobFileAppender.getLogPath()).listFiles(); |
||||
|
if (childDirs!=null && childDirs.length>0) { |
||||
|
|
||||
|
// today
|
||||
|
Calendar todayCal = Calendar.getInstance(); |
||||
|
todayCal.set(Calendar.HOUR_OF_DAY,0); |
||||
|
todayCal.set(Calendar.MINUTE,0); |
||||
|
todayCal.set(Calendar.SECOND,0); |
||||
|
todayCal.set(Calendar.MILLISECOND,0); |
||||
|
|
||||
|
Date todayDate = todayCal.getTime(); |
||||
|
|
||||
|
for (File childFile: childDirs) { |
||||
|
|
||||
|
// valid
|
||||
|
if (!childFile.isDirectory()) { |
||||
|
continue; |
||||
|
} |
||||
|
if (childFile.getName().indexOf("-") == -1) { |
||||
|
continue; |
||||
|
} |
||||
|
|
||||
|
// file create date
|
||||
|
Date logFileCreateDate = null; |
||||
|
try { |
||||
|
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); |
||||
|
logFileCreateDate = simpleDateFormat.parse(childFile.getName()); |
||||
|
} catch (ParseException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
if (logFileCreateDate == null) { |
||||
|
continue; |
||||
|
} |
||||
|
|
||||
|
if ((todayDate.getTime()-logFileCreateDate.getTime()) >= logRetentionDays * (24 * 60 * 60 * 1000) ) { |
||||
|
FileUtil.deleteRecursively(childFile); |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} catch (Exception e) { |
||||
|
if (!toStop) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
try { |
||||
|
TimeUnit.DAYS.sleep(1); |
||||
|
} catch (InterruptedException e) { |
||||
|
if (!toStop) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
logger.info(">>>>>>>>>>> xxl-job, executor JobLogFileCleanThread thread destroy."); |
||||
|
|
||||
|
} |
||||
|
}); |
||||
|
localThread.setDaemon(true); |
||||
|
localThread.setName("xxl-job, executor JobLogFileCleanThread"); |
||||
|
localThread.start(); |
||||
|
} |
||||
|
|
||||
|
public void toStop() { |
||||
|
toStop = true; |
||||
|
|
||||
|
if (localThread == null) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// interrupt and wait
|
||||
|
localThread.interrupt(); |
||||
|
try { |
||||
|
localThread.join(); |
||||
|
} catch (InterruptedException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,238 @@ |
|||||
|
package io.sc.platform.job.executor.thread; |
||||
|
|
||||
|
import io.sc.platform.job.executor.*; |
||||
|
import io.sc.platform.job.executor.manager.JobThreadManager; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
import java.io.PrintWriter; |
||||
|
import java.io.StringWriter; |
||||
|
import java.util.Collections; |
||||
|
import java.util.Date; |
||||
|
import java.util.HashSet; |
||||
|
import java.util.Set; |
||||
|
import java.util.concurrent.*; |
||||
|
|
||||
|
public class JobThread extends Thread{ |
||||
|
private static Logger logger = LoggerFactory.getLogger(JobThread.class); |
||||
|
|
||||
|
private String jobId; |
||||
|
private JobHandler handler; |
||||
|
private LinkedBlockingQueue<TriggerParam> triggerQueue; |
||||
|
private Set<Long> triggerLogIdSet; // avoid repeat trigger for the same TRIGGER_LOG_ID
|
||||
|
|
||||
|
private volatile boolean toStop = false; |
||||
|
private String stopReason; |
||||
|
|
||||
|
private boolean running = false; // if running job
|
||||
|
private int idleTimes = 0; // idel times
|
||||
|
|
||||
|
|
||||
|
public JobThread(String jobId, JobHandler handler) { |
||||
|
this.jobId = jobId; |
||||
|
this.handler = handler; |
||||
|
this.triggerQueue = new LinkedBlockingQueue<TriggerParam>(); |
||||
|
this.triggerLogIdSet = Collections.synchronizedSet(new HashSet<Long>()); |
||||
|
|
||||
|
// assign job thread name
|
||||
|
this.setName("xxl-job, JobThread-"+jobId+"-"+System.currentTimeMillis()); |
||||
|
} |
||||
|
public JobHandler getHandler() { |
||||
|
return handler; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* new trigger to queue |
||||
|
* |
||||
|
* @param triggerParam |
||||
|
* @return |
||||
|
*/ |
||||
|
public boolean pushTriggerQueue(TriggerParam triggerParam) { |
||||
|
if (triggerLogIdSet.contains(triggerParam.getLogId())) { |
||||
|
logger.warn("repeate trigger job, logId:{}", triggerParam.getLogId()); |
||||
|
return false; |
||||
|
} |
||||
|
triggerLogIdSet.add(triggerParam.getLogId()); |
||||
|
triggerQueue.add(triggerParam); |
||||
|
return true; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* kill job thread |
||||
|
* |
||||
|
* @param stopReason |
||||
|
*/ |
||||
|
public void toStop(String stopReason) { |
||||
|
/** |
||||
|
* Thread.interrupt只支持终止线程的阻塞状态(wait、join、sleep)， |
||||
|
* 在阻塞出抛出InterruptedException异常,但是并不会终止运行的线程本身； |
||||
|
* 所以需要注意，此处彻底销毁本线程，需要通过共享变量方式； |
||||
|
*/ |
||||
|
this.toStop = true; |
||||
|
this.stopReason = stopReason; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* is running job |
||||
|
* @return |
||||
|
*/ |
||||
|
public boolean isRunningOrHasQueue() { |
||||
|
return running || triggerQueue.size()>0; |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void run() { |
||||
|
// init
|
||||
|
try { |
||||
|
handler.init(); |
||||
|
} catch (Throwable e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
|
||||
|
// execute
|
||||
|
while(!toStop){ |
||||
|
running = false; |
||||
|
idleTimes++; |
||||
|
|
||||
|
TriggerParam triggerParam = null; |
||||
|
try { |
||||
|
// to check toStop signal, we need cycle, so wo cannot use queue.take(), instand of poll(timeout)
|
||||
|
triggerParam = triggerQueue.poll(3L, TimeUnit.SECONDS); |
||||
|
if (triggerParam!=null) { |
||||
|
running = true; |
||||
|
idleTimes = 0; |
||||
|
triggerLogIdSet.remove(triggerParam.getLogId()); |
||||
|
|
||||
|
// log filename, like "logPath/yyyy-MM-dd/9999.log"
|
||||
|
String logFileName = XxlJobFileAppender.makeLogFileName(new Date(triggerParam.getLogDateTime()), triggerParam.getLogId()); |
||||
|
XxlJobContext xxlJobContext = new XxlJobContext( |
||||
|
triggerParam.getJobId(), |
||||
|
triggerParam.getExecutorParams(), |
||||
|
logFileName, |
||||
|
triggerParam.getBroadcastIndex(), |
||||
|
triggerParam.getBroadcastTotal()); |
||||
|
|
||||
|
// init job context
|
||||
|
XxlJobContext.setXxlJobContext(xxlJobContext); |
||||
|
|
||||
|
// execute
|
||||
|
XxlJobHelper.log("<br>----------- xxl-job job execute start -----------<br>----------- Param:" + xxlJobContext.getJobParam()); |
||||
|
|
||||
|
if (triggerParam.getExecutorTimeout() > 0) { |
||||
|
// limit timeout
|
||||
|
Thread futureThread = null; |
||||
|
try { |
||||
|
FutureTask<Boolean> futureTask = new FutureTask<Boolean>(new Callable<Boolean>() { |
||||
|
@Override |
||||
|
public Boolean call() throws Exception { |
||||
|
|
||||
|
// init job context
|
||||
|
XxlJobContext.setXxlJobContext(xxlJobContext); |
||||
|
|
||||
|
handler.execute(); |
||||
|
return true; |
||||
|
} |
||||
|
}); |
||||
|
futureThread = new Thread(futureTask); |
||||
|
futureThread.start(); |
||||
|
|
||||
|
Boolean tempResult = futureTask.get(triggerParam.getExecutorTimeout(), TimeUnit.SECONDS); |
||||
|
} catch (TimeoutException e) { |
||||
|
|
||||
|
XxlJobHelper.log("<br>----------- xxl-job job execute timeout"); |
||||
|
XxlJobHelper.log(e); |
||||
|
|
||||
|
// handle result
|
||||
|
XxlJobHelper.handleTimeout("job execute timeout "); |
||||
|
} finally { |
||||
|
futureThread.interrupt(); |
||||
|
} |
||||
|
} else { |
||||
|
// just execute
|
||||
|
handler.execute(); |
||||
|
} |
||||
|
|
||||
|
// valid execute handle data
|
||||
|
if (XxlJobContext.getXxlJobContext().getHandleCode() <= 0) { |
||||
|
XxlJobHelper.handleFail("job handle result lost."); |
||||
|
} else { |
||||
|
String tempHandleMsg = XxlJobContext.getXxlJobContext().getHandleMsg(); |
||||
|
tempHandleMsg = (tempHandleMsg!=null&&tempHandleMsg.length()>50000) |
||||
|
?tempHandleMsg.substring(0, 50000).concat("...") |
||||
|
:tempHandleMsg; |
||||
|
XxlJobContext.getXxlJobContext().setHandleMsg(tempHandleMsg); |
||||
|
} |
||||
|
XxlJobHelper.log("<br>----------- xxl-job job execute end(finish) -----------<br>----------- Result: handleCode=" |
||||
|
+ XxlJobContext.getXxlJobContext().getHandleCode() |
||||
|
+ ", handleMsg = " |
||||
|
+ XxlJobContext.getXxlJobContext().getHandleMsg() |
||||
|
); |
||||
|
|
||||
|
} else { |
||||
|
if (idleTimes > 30) { |
||||
|
if(triggerQueue.size() == 0) { // avoid concurrent trigger causes jobId-lost
|
||||
|
JobThreadManager.removeJobThread(jobId, "excutor idel times over limit."); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} catch (Throwable e) { |
||||
|
if (toStop) { |
||||
|
XxlJobHelper.log("<br>----------- JobThread toStop, stopReason:" + stopReason); |
||||
|
} |
||||
|
|
||||
|
// handle result
|
||||
|
StringWriter stringWriter = new StringWriter(); |
||||
|
e.printStackTrace(new PrintWriter(stringWriter)); |
||||
|
String errorMsg = stringWriter.toString(); |
||||
|
|
||||
|
XxlJobHelper.handleFail(errorMsg); |
||||
|
|
||||
|
XxlJobHelper.log("<br>----------- JobThread Exception:" + errorMsg + "<br>----------- xxl-job job execute end(error) -----------"); |
||||
|
} finally { |
||||
|
if(triggerParam != null) { |
||||
|
// callback handler info
|
||||
|
if (!toStop) { |
||||
|
// commonm
|
||||
|
TriggerCallbackThread.pushCallBack(new HandleCallbackParam( |
||||
|
triggerParam.getLogId(), |
||||
|
triggerParam.getLogDateTime(), |
||||
|
XxlJobContext.getXxlJobContext().getHandleCode(), |
||||
|
XxlJobContext.getXxlJobContext().getHandleMsg() ) |
||||
|
); |
||||
|
} else { |
||||
|
// is killed
|
||||
|
TriggerCallbackThread.pushCallBack(new HandleCallbackParam( |
||||
|
triggerParam.getLogId(), |
||||
|
triggerParam.getLogDateTime(), |
||||
|
XxlJobContext.HANDLE_CODE_FAIL, |
||||
|
stopReason + " [job running, killed]" ) |
||||
|
); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// callback trigger request in queue
|
||||
|
while(triggerQueue !=null && triggerQueue.size()>0){ |
||||
|
TriggerParam triggerParam = triggerQueue.poll(); |
||||
|
if (triggerParam!=null) { |
||||
|
// is killed
|
||||
|
TriggerCallbackThread.pushCallBack(new HandleCallbackParam( |
||||
|
triggerParam.getLogId(), |
||||
|
triggerParam.getLogDateTime(), |
||||
|
XxlJobContext.HANDLE_CODE_FAIL, |
||||
|
stopReason + " [job not executed, in the job queue, killed.]") |
||||
|
); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// destroy
|
||||
|
try { |
||||
|
handler.destroy(); |
||||
|
} catch (Throwable e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
|
||||
|
logger.info(">>>>>>>>>>> xxl-job JobThread stoped, hashCode:{}", Thread.currentThread()); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,254 @@ |
|||||
|
package io.sc.platform.job.executor.thread; |
||||
|
|
||||
|
import io.sc.platform.job.executor.*; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.util.ArrayList; |
||||
|
import java.util.Date; |
||||
|
import java.util.List; |
||||
|
import java.util.concurrent.LinkedBlockingQueue; |
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
|
||||
|
/** |
||||
|
* Created by xuxueli on 16/7/22. |
||||
|
*/ |
||||
|
public class TriggerCallbackThread { |
||||
|
private static Logger logger = LoggerFactory.getLogger(TriggerCallbackThread.class); |
||||
|
|
||||
|
private static TriggerCallbackThread instance = new TriggerCallbackThread(); |
||||
|
public static TriggerCallbackThread getInstance(){ |
||||
|
return instance; |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* job results callback queue |
||||
|
*/ |
||||
|
private LinkedBlockingQueue<HandleCallbackParam> callBackQueue = new LinkedBlockingQueue<HandleCallbackParam>(); |
||||
|
public static void pushCallBack(HandleCallbackParam callback){ |
||||
|
getInstance().callBackQueue.add(callback); |
||||
|
logger.debug(">>>>>>>>>>> xxl-job, push callback request, logId:{}", callback.getLogId()); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* callback thread |
||||
|
*/ |
||||
|
private Thread triggerCallbackThread; |
||||
|
private Thread triggerRetryCallbackThread; |
||||
|
private volatile boolean toStop = false; |
||||
|
public void start() { |
||||
|
|
||||
|
// valid
|
||||
|
if (XxlJobExecutor.getAdminBizList() == null) { |
||||
|
logger.warn(">>>>>>>>>>> xxl-job, executor callback config fail, adminAddresses is null."); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// callback
|
||||
|
triggerCallbackThread = new Thread(new Runnable() { |
||||
|
|
||||
|
@Override |
||||
|
public void run() { |
||||
|
|
||||
|
// normal callback
|
||||
|
while(!toStop){ |
||||
|
try { |
||||
|
HandleCallbackParam callback = getInstance().callBackQueue.take(); |
||||
|
if (callback != null) { |
||||
|
|
||||
|
// callback list param
|
||||
|
List<HandleCallbackParam> callbackParamList = new ArrayList<HandleCallbackParam>(); |
||||
|
int drainToNum = getInstance().callBackQueue.drainTo(callbackParamList); |
||||
|
callbackParamList.add(callback); |
||||
|
|
||||
|
// callback, will retry if error
|
||||
|
if (callbackParamList!=null && callbackParamList.size()>0) { |
||||
|
doCallback(callbackParamList); |
||||
|
} |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
if (!toStop) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// last callback
|
||||
|
try { |
||||
|
List<HandleCallbackParam> callbackParamList = new ArrayList<HandleCallbackParam>(); |
||||
|
int drainToNum = getInstance().callBackQueue.drainTo(callbackParamList); |
||||
|
if (callbackParamList!=null && callbackParamList.size()>0) { |
||||
|
doCallback(callbackParamList); |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
if (!toStop) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
logger.info(">>>>>>>>>>> xxl-job, executor callback thread destroy."); |
||||
|
|
||||
|
} |
||||
|
}); |
||||
|
triggerCallbackThread.setDaemon(true); |
||||
|
triggerCallbackThread.setName("xxl-job, executor TriggerCallbackThread"); |
||||
|
triggerCallbackThread.start(); |
||||
|
|
||||
|
|
||||
|
// retry
|
||||
|
triggerRetryCallbackThread = new Thread(new Runnable() { |
||||
|
@Override |
||||
|
public void run() { |
||||
|
while(!toStop){ |
||||
|
try { |
||||
|
retryFailCallbackFile(); |
||||
|
} catch (Exception e) { |
||||
|
if (!toStop) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
try { |
||||
|
TimeUnit.SECONDS.sleep(RegistryConfig.BEAT_TIMEOUT); |
||||
|
} catch (InterruptedException e) { |
||||
|
if (!toStop) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
logger.info(">>>>>>>>>>> xxl-job, executor retry callback thread destroy."); |
||||
|
} |
||||
|
}); |
||||
|
triggerRetryCallbackThread.setDaemon(true); |
||||
|
triggerRetryCallbackThread.start(); |
||||
|
|
||||
|
} |
||||
|
public void toStop(){ |
||||
|
toStop = true; |
||||
|
// stop callback, interrupt and wait
|
||||
|
if (triggerCallbackThread != null) { // support empty admin address
|
||||
|
triggerCallbackThread.interrupt(); |
||||
|
try { |
||||
|
triggerCallbackThread.join(); |
||||
|
} catch (InterruptedException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// stop retry, interrupt and wait
|
||||
|
if (triggerRetryCallbackThread != null) { |
||||
|
triggerRetryCallbackThread.interrupt(); |
||||
|
try { |
||||
|
triggerRetryCallbackThread.join(); |
||||
|
} catch (InterruptedException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* do callback, will retry if error |
||||
|
* @param callbackParamList |
||||
|
*/ |
||||
|
private void doCallback(List<HandleCallbackParam> callbackParamList){ |
||||
|
/* |
||||
|
boolean callbackRet = false; |
||||
|
// callback, will retry if error
|
||||
|
for (AdminBiz adminBiz: XxlJobExecutor.getAdminBizList()) { |
||||
|
try { |
||||
|
ReturnT<String> callbackResult = adminBiz.callback(callbackParamList); |
||||
|
if (callbackResult!=null && ReturnT.SUCCESS_CODE == callbackResult.getCode()) { |
||||
|
callbackLog(callbackParamList, "<br>----------- xxl-job job callback finish."); |
||||
|
callbackRet = true; |
||||
|
break; |
||||
|
} else { |
||||
|
callbackLog(callbackParamList, "<br>----------- xxl-job job callback fail, callbackResult:" + callbackResult); |
||||
|
} |
||||
|
} catch (Exception e) { |
||||
|
callbackLog(callbackParamList, "<br>----------- xxl-job job callback error, errorMsg:" + e.getMessage()); |
||||
|
} |
||||
|
} |
||||
|
if (!callbackRet) { |
||||
|
appendFailCallbackFile(callbackParamList); |
||||
|
}*/ |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* callback log |
||||
|
*/ |
||||
|
private void callbackLog(List<HandleCallbackParam> callbackParamList, String logContent){ |
||||
|
for (HandleCallbackParam callbackParam: callbackParamList) { |
||||
|
String logFileName = XxlJobFileAppender.makeLogFileName(new Date(callbackParam.getLogDateTim()), callbackParam.getLogId()); |
||||
|
XxlJobContext.setXxlJobContext(new XxlJobContext( |
||||
|
-1, |
||||
|
null, |
||||
|
logFileName, |
||||
|
-1, |
||||
|
-1)); |
||||
|
XxlJobHelper.log(logContent); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
// ---------------------- fail-callback file ----------------------
|
||||
|
|
||||
|
private static String failCallbackFilePath = XxlJobFileAppender.getLogPath().concat(File.separator).concat("callbacklog").concat(File.separator); |
||||
|
private static String failCallbackFileName = failCallbackFilePath.concat("xxl-job-callback-{x}").concat(".log"); |
||||
|
|
||||
|
private void appendFailCallbackFile(List<HandleCallbackParam> callbackParamList){ |
||||
|
/* |
||||
|
// valid
|
||||
|
if (callbackParamList==null || callbackParamList.size()==0) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// append file
|
||||
|
byte[] callbackParamList_bytes = JdkSerializeTool.serialize(callbackParamList); |
||||
|
|
||||
|
File callbackLogFile = new File(failCallbackFileName.replace("{x}", String.valueOf(System.currentTimeMillis()))); |
||||
|
if (callbackLogFile.exists()) { |
||||
|
for (int i = 0; i < 100; i++) { |
||||
|
callbackLogFile = new File(failCallbackFileName.replace("{x}", String.valueOf(System.currentTimeMillis()).concat("-").concat(String.valueOf(i)) )); |
||||
|
if (!callbackLogFile.exists()) { |
||||
|
break; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
FileUtil.writeFileContent(callbackLogFile, callbackParamList_bytes); |
||||
|
*/ |
||||
|
} |
||||
|
|
||||
|
private void retryFailCallbackFile(){ |
||||
|
/* |
||||
|
// valid
|
||||
|
File callbackLogPath = new File(failCallbackFilePath); |
||||
|
if (!callbackLogPath.exists()) { |
||||
|
return; |
||||
|
} |
||||
|
if (callbackLogPath.isFile()) { |
||||
|
callbackLogPath.delete(); |
||||
|
} |
||||
|
if (!(callbackLogPath.isDirectory() && callbackLogPath.list()!=null && callbackLogPath.list().length>0)) { |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// load and clear file, retry
|
||||
|
for (File callbaclLogFile: callbackLogPath.listFiles()) { |
||||
|
byte[] callbackParamList_bytes = FileUtil.readFileContent(callbaclLogFile); |
||||
|
|
||||
|
// avoid empty file
|
||||
|
if(callbackParamList_bytes == null || callbackParamList_bytes.length < 1){ |
||||
|
callbaclLogFile.delete(); |
||||
|
continue; |
||||
|
} |
||||
|
|
||||
|
List<HandleCallbackParam> callbackParamList = (List<HandleCallbackParam>) JdkSerializeTool.deserialize(callbackParamList_bytes, List.class); |
||||
|
|
||||
|
callbaclLogFile.delete(); |
||||
|
doCallback(callbackParamList); |
||||
|
} |
||||
|
*/ |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,156 @@ |
|||||
|
package io.sc.platform.job.executor.util; |
||||
|
|
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
import java.text.DateFormat; |
||||
|
import java.text.ParseException; |
||||
|
import java.text.SimpleDateFormat; |
||||
|
import java.util.Calendar; |
||||
|
import java.util.Date; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.Map; |
||||
|
|
||||
|
/** |
||||
|
* date util |
||||
|
* |
||||
|
* @author xuxueli 2018-08-19 01:24:11 |
||||
|
*/ |
||||
|
public class DateUtil { |
||||
|
|
||||
|
// ---------------------- format parse ----------------------
|
||||
|
private static Logger logger = LoggerFactory.getLogger(DateUtil.class); |
||||
|
|
||||
|
private static final String DATE_FORMAT = "yyyy-MM-dd"; |
||||
|
private static final String DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; |
||||
|
|
||||
|
private static final ThreadLocal<Map<String, DateFormat>> dateFormatThreadLocal = new ThreadLocal<Map<String, DateFormat>>(); |
||||
|
private static DateFormat getDateFormat(String pattern) { |
||||
|
if (pattern==null || pattern.trim().length()==0) { |
||||
|
throw new IllegalArgumentException("pattern cannot be empty."); |
||||
|
} |
||||
|
|
||||
|
Map<String, DateFormat> dateFormatMap = dateFormatThreadLocal.get(); |
||||
|
if(dateFormatMap!=null && dateFormatMap.containsKey(pattern)){ |
||||
|
return dateFormatMap.get(pattern); |
||||
|
} |
||||
|
|
||||
|
synchronized (dateFormatThreadLocal) { |
||||
|
if (dateFormatMap == null) { |
||||
|
dateFormatMap = new HashMap<String, DateFormat>(); |
||||
|
} |
||||
|
dateFormatMap.put(pattern, new SimpleDateFormat(pattern)); |
||||
|
dateFormatThreadLocal.set(dateFormatMap); |
||||
|
} |
||||
|
|
||||
|
return dateFormatMap.get(pattern); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* format datetime. like "yyyy-MM-dd" |
||||
|
* |
||||
|
* @param date |
||||
|
* @return |
||||
|
* @throws ParseException |
||||
|
*/ |
||||
|
public static String formatDate(Date date) { |
||||
|
return format(date, DATE_FORMAT); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* format date. like "yyyy-MM-dd HH:mm:ss" |
||||
|
* |
||||
|
* @param date |
||||
|
* @return |
||||
|
* @throws ParseException |
||||
|
*/ |
||||
|
public static String formatDateTime(Date date) { |
||||
|
return format(date, DATETIME_FORMAT); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* format date |
||||
|
* |
||||
|
* @param date |
||||
|
* @param patten |
||||
|
* @return |
||||
|
* @throws ParseException |
||||
|
*/ |
||||
|
public static String format(Date date, String patten) { |
||||
|
return getDateFormat(patten).format(date); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* parse date string, like "yyyy-MM-dd HH:mm:s" |
||||
|
* |
||||
|
* @param dateString |
||||
|
* @return |
||||
|
* @throws ParseException |
||||
|
*/ |
||||
|
public static Date parseDate(String dateString){ |
||||
|
return parse(dateString, DATE_FORMAT); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* parse datetime string, like "yyyy-MM-dd HH:mm:ss" |
||||
|
* |
||||
|
* @param dateString |
||||
|
* @return |
||||
|
* @throws ParseException |
||||
|
*/ |
||||
|
public static Date parseDateTime(String dateString) { |
||||
|
return parse(dateString, DATETIME_FORMAT); |
||||
|
} |
||||
|
|
||||
|
/** |
||||
|
* parse date |
||||
|
* |
||||
|
* @param dateString |
||||
|
* @param pattern |
||||
|
* @return |
||||
|
* @throws ParseException |
||||
|
*/ |
||||
|
public static Date parse(String dateString, String pattern) { |
||||
|
try { |
||||
|
Date date = getDateFormat(pattern).parse(dateString); |
||||
|
return date; |
||||
|
} catch (Exception e) { |
||||
|
logger.warn("parse date error, dateString = {}, pattern={}; errorMsg = {}", dateString, pattern, e.getMessage()); |
||||
|
return null; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
// ---------------------- add date ----------------------
|
||||
|
|
||||
|
public static Date addYears(final Date date, final int amount) { |
||||
|
return add(date, Calendar.YEAR, amount); |
||||
|
} |
||||
|
|
||||
|
public static Date addMonths(final Date date, final int amount) { |
||||
|
return add(date, Calendar.MONTH, amount); |
||||
|
} |
||||
|
|
||||
|
public static Date addDays(final Date date, final int amount) { |
||||
|
return add(date, Calendar.DAY_OF_MONTH, amount); |
||||
|
} |
||||
|
|
||||
|
public static Date addHours(final Date date, final int amount) { |
||||
|
return add(date, Calendar.HOUR_OF_DAY, amount); |
||||
|
} |
||||
|
|
||||
|
public static Date addMinutes(final Date date, final int amount) { |
||||
|
return add(date, Calendar.MINUTE, amount); |
||||
|
} |
||||
|
|
||||
|
private static Date add(final Date date, final int calendarField, final int amount) { |
||||
|
if (date == null) { |
||||
|
return null; |
||||
|
} |
||||
|
final Calendar c = Calendar.getInstance(); |
||||
|
c.setTime(date); |
||||
|
c.add(calendarField, amount); |
||||
|
return c.getTime(); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,181 @@ |
|||||
|
package io.sc.platform.job.executor.util; |
||||
|
|
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.io.FileInputStream; |
||||
|
import java.io.FileOutputStream; |
||||
|
import java.io.IOException; |
||||
|
|
||||
|
/** |
||||
|
* file tool |
||||
|
* |
||||
|
* @author xuxueli 2017-12-29 17:56:48 |
||||
|
*/ |
||||
|
public class FileUtil { |
||||
|
private static Logger logger = LoggerFactory.getLogger(FileUtil.class); |
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* delete recursively |
||||
|
* |
||||
|
* @param root |
||||
|
* @return |
||||
|
*/ |
||||
|
public static boolean deleteRecursively(File root) { |
||||
|
if (root != null && root.exists()) { |
||||
|
if (root.isDirectory()) { |
||||
|
File[] children = root.listFiles(); |
||||
|
if (children != null) { |
||||
|
for (File child : children) { |
||||
|
deleteRecursively(child); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return root.delete(); |
||||
|
} |
||||
|
return false; |
||||
|
} |
||||
|
|
||||
|
|
||||
|
public static void deleteFile(String fileName) { |
||||
|
// file
|
||||
|
File file = new File(fileName); |
||||
|
if (file.exists()) { |
||||
|
file.delete(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
public static void writeFileContent(File file, byte[] data) { |
||||
|
|
||||
|
// file
|
||||
|
if (!file.exists()) { |
||||
|
file.getParentFile().mkdirs(); |
||||
|
} |
||||
|
|
||||
|
// append file content
|
||||
|
FileOutputStream fos = null; |
||||
|
try { |
||||
|
fos = new FileOutputStream(file); |
||||
|
fos.write(data); |
||||
|
fos.flush(); |
||||
|
} catch (Exception e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} finally { |
||||
|
if (fos != null) { |
||||
|
try { |
||||
|
fos.close(); |
||||
|
} catch (IOException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
public static byte[] readFileContent(File file) { |
||||
|
Long filelength = file.length(); |
||||
|
byte[] filecontent = new byte[filelength.intValue()]; |
||||
|
|
||||
|
FileInputStream in = null; |
||||
|
try { |
||||
|
in = new FileInputStream(file); |
||||
|
in.read(filecontent); |
||||
|
in.close(); |
||||
|
|
||||
|
return filecontent; |
||||
|
} catch (Exception e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
return null; |
||||
|
} finally { |
||||
|
if (in != null) { |
||||
|
try { |
||||
|
in.close(); |
||||
|
} catch (IOException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
/*public static void appendFileLine(String fileName, String content) { |
||||
|
|
||||
|
// file
|
||||
|
File file = new File(fileName); |
||||
|
if (!file.exists()) { |
||||
|
try { |
||||
|
file.createNewFile(); |
||||
|
} catch (IOException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
return; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
// content
|
||||
|
if (content == null) { |
||||
|
content = ""; |
||||
|
} |
||||
|
content += "\r\n"; |
||||
|
|
||||
|
// append file content
|
||||
|
FileOutputStream fos = null; |
||||
|
try { |
||||
|
fos = new FileOutputStream(file, true); |
||||
|
fos.write(content.getBytes("utf-8")); |
||||
|
fos.flush(); |
||||
|
} catch (Exception e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} finally { |
||||
|
if (fos != null) { |
||||
|
try { |
||||
|
fos.close(); |
||||
|
} catch (IOException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
|
|
||||
|
public static List<String> loadFileLines(String fileName){ |
||||
|
|
||||
|
List<String> result = new ArrayList<>(); |
||||
|
|
||||
|
// valid log file
|
||||
|
File file = new File(fileName); |
||||
|
if (!file.exists()) { |
||||
|
return result; |
||||
|
} |
||||
|
|
||||
|
// read file
|
||||
|
StringBuffer logContentBuffer = new StringBuffer(); |
||||
|
int toLineNum = 0; |
||||
|
LineNumberReader reader = null; |
||||
|
try { |
||||
|
//reader = new LineNumberReader(new FileReader(logFile));
|
||||
|
reader = new LineNumberReader(new InputStreamReader(new FileInputStream(file), "utf-8")); |
||||
|
String line = null; |
||||
|
while ((line = reader.readLine())!=null) { |
||||
|
if (line!=null && line.trim().length()>0) { |
||||
|
result.add(line); |
||||
|
} |
||||
|
} |
||||
|
} catch (IOException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} finally { |
||||
|
if (reader != null) { |
||||
|
try { |
||||
|
reader.close(); |
||||
|
} catch (IOException e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return result; |
||||
|
}*/ |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,158 @@ |
|||||
|
package io.sc.platform.job.executor.util; |
||||
|
|
||||
|
import io.sc.platform.job.executor.ReturnT; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
|
||||
|
import javax.net.ssl.*; |
||||
|
import java.io.BufferedReader; |
||||
|
import java.io.DataOutputStream; |
||||
|
import java.io.InputStreamReader; |
||||
|
import java.net.HttpURLConnection; |
||||
|
import java.net.URL; |
||||
|
import java.security.cert.CertificateException; |
||||
|
import java.security.cert.X509Certificate; |
||||
|
|
||||
|
/** |
||||
|
* @author xuxueli 2018-11-25 00:55:31 |
||||
|
*/ |
||||
|
public class XxlJobRemotingUtil { |
||||
|
private static Logger logger = LoggerFactory.getLogger(XxlJobRemotingUtil.class); |
||||
|
public static final String XXL_JOB_ACCESS_TOKEN = "XXL-JOB-ACCESS-TOKEN"; |
||||
|
|
||||
|
|
||||
|
// trust-https start
|
||||
|
private static void trustAllHosts(HttpsURLConnection connection) { |
||||
|
try { |
||||
|
SSLContext sc = SSLContext.getInstance("TLS"); |
||||
|
sc.init(null, trustAllCerts, new java.security.SecureRandom()); |
||||
|
SSLSocketFactory newFactory = sc.getSocketFactory(); |
||||
|
|
||||
|
connection.setSSLSocketFactory(newFactory); |
||||
|
} catch (Exception e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
} |
||||
|
connection.setHostnameVerifier(new HostnameVerifier() { |
||||
|
@Override |
||||
|
public boolean verify(String hostname, SSLSession session) { |
||||
|
return true; |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
private static final TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() { |
||||
|
@Override |
||||
|
public X509Certificate[] getAcceptedIssuers() { |
||||
|
return new X509Certificate[]{}; |
||||
|
} |
||||
|
@Override |
||||
|
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { |
||||
|
} |
||||
|
@Override |
||||
|
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { |
||||
|
} |
||||
|
}}; |
||||
|
// trust-https end
|
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* post |
||||
|
* |
||||
|
* @param url |
||||
|
* @param accessToken |
||||
|
* @param timeout |
||||
|
* @param requestObj |
||||
|
* @param returnTargClassOfT |
||||
|
* @return |
||||
|
*/ |
||||
|
public static ReturnT postBody(String url, String accessToken, int timeout, Object requestObj, Class returnTargClassOfT) { |
||||
|
HttpURLConnection connection = null; |
||||
|
BufferedReader bufferedReader = null; |
||||
|
try { |
||||
|
// connection
|
||||
|
URL realUrl = new URL(url); |
||||
|
connection = (HttpURLConnection) realUrl.openConnection(); |
||||
|
|
||||
|
// trust-https
|
||||
|
boolean useHttps = url.startsWith("https"); |
||||
|
if (useHttps) { |
||||
|
HttpsURLConnection https = (HttpsURLConnection) connection; |
||||
|
trustAllHosts(https); |
||||
|
} |
||||
|
|
||||
|
// connection setting
|
||||
|
connection.setRequestMethod("POST"); |
||||
|
connection.setDoOutput(true); |
||||
|
connection.setDoInput(true); |
||||
|
connection.setUseCaches(false); |
||||
|
connection.setReadTimeout(timeout * 1000); |
||||
|
connection.setConnectTimeout(3 * 1000); |
||||
|
connection.setRequestProperty("connection", "Keep-Alive"); |
||||
|
connection.setRequestProperty("Content-Type", "application/json;charset=UTF-8"); |
||||
|
connection.setRequestProperty("Accept-Charset", "application/json;charset=UTF-8"); |
||||
|
|
||||
|
if(accessToken!=null && accessToken.trim().length()>0){ |
||||
|
connection.setRequestProperty(XXL_JOB_ACCESS_TOKEN, accessToken); |
||||
|
} |
||||
|
|
||||
|
// do connection
|
||||
|
connection.connect(); |
||||
|
|
||||
|
// write requestBody
|
||||
|
if (requestObj != null) { |
||||
|
String requestBody = "GsonTool.toJson(requestObj)"; |
||||
|
|
||||
|
DataOutputStream dataOutputStream = new DataOutputStream(connection.getOutputStream()); |
||||
|
dataOutputStream.write(requestBody.getBytes("UTF-8")); |
||||
|
dataOutputStream.flush(); |
||||
|
dataOutputStream.close(); |
||||
|
} |
||||
|
|
||||
|
/*byte[] requestBodyBytes = requestBody.getBytes("UTF-8"); |
||||
|
connection.setRequestProperty("Content-Length", String.valueOf(requestBodyBytes.length)); |
||||
|
OutputStream outwritestream = connection.getOutputStream(); |
||||
|
outwritestream.write(requestBodyBytes); |
||||
|
outwritestream.flush(); |
||||
|
outwritestream.close();*/ |
||||
|
|
||||
|
// valid StatusCode
|
||||
|
int statusCode = connection.getResponseCode(); |
||||
|
if (statusCode != 200) { |
||||
|
return new ReturnT<String>(ReturnT.FAIL_CODE, "xxl-job remoting fail, StatusCode("+ statusCode +") invalid. for url : " + url); |
||||
|
} |
||||
|
|
||||
|
// result
|
||||
|
bufferedReader = new BufferedReader(new InputStreamReader(connection.getInputStream(), "UTF-8")); |
||||
|
StringBuilder result = new StringBuilder(); |
||||
|
String line; |
||||
|
while ((line = bufferedReader.readLine()) != null) { |
||||
|
result.append(line); |
||||
|
} |
||||
|
String resultJson = result.toString(); |
||||
|
|
||||
|
// parse returnT
|
||||
|
try { |
||||
|
ReturnT returnT = null;//GsonTool.fromJson(resultJson, ReturnT.class, returnTargClassOfT);
|
||||
|
return returnT; |
||||
|
} catch (Exception e) { |
||||
|
logger.error("xxl-job remoting (url="+url+") response content invalid("+ resultJson +").", e); |
||||
|
return new ReturnT<String>(ReturnT.FAIL_CODE, "xxl-job remoting (url="+url+") response content invalid("+ resultJson +")."); |
||||
|
} |
||||
|
|
||||
|
} catch (Exception e) { |
||||
|
logger.error(e.getMessage(), e); |
||||
|
return new ReturnT<String>(ReturnT.FAIL_CODE, "xxl-job remoting error("+ e.getMessage() +"), for url : " + url); |
||||
|
} finally { |
||||
|
try { |
||||
|
if (bufferedReader != null) { |
||||
|
bufferedReader.close(); |
||||
|
} |
||||
|
if (connection != null) { |
||||
|
connection.disconnect(); |
||||
|
} |
||||
|
} catch (Exception e2) { |
||||
|
logger.error(e2.getMessage(), e2); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,10 @@ |
|||||
|
[ |
||||
|
{ |
||||
|
"module" : "io.sc.platform.job.executor", |
||||
|
"order" : 10000, |
||||
|
"description": "job manager configuration", |
||||
|
"properties": [ |
||||
|
"job.manager.urls[0]=http://localhost:8080/api/job/manager/executor" |
||||
|
] |
||||
|
} |
||||
|
] |
||||
@ -0,0 +1,7 @@ |
|||||
|
{ |
||||
|
"includes":[ |
||||
|
"io.sc.platform.job.executor.component", |
||||
|
"io.sc.platform.job.executor.client.impl", |
||||
|
"io.sc.platform.job.executor.service.impl" |
||||
|
] |
||||
|
} |
||||
@ -0,0 +1,3 @@ |
|||||
|
# Auto Configuration |
||||
|
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ |
||||
|
io.sc.platform.job.executor.configure.ExecutorConfigurationAutoConfiguration |
||||
@ -0,0 +1,23 @@ |
|||||
|
package io.sc.platform.job.manager.controller; |
||||
|
|
||||
|
import io.sc.platform.job.core.TaskLog; |
||||
|
import io.sc.platform.job.manager.jpa.entity.TaskLogEntity; |
||||
|
import io.sc.platform.job.manager.jpa.repository.TaskLogRepository; |
||||
|
import io.sc.platform.job.manager.service.TaskLogService; |
||||
|
import io.sc.platform.mvc.controller.support.RestCrudController; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.web.bind.annotation.PostMapping; |
||||
|
import org.springframework.web.bind.annotation.RequestBody; |
||||
|
import org.springframework.web.bind.annotation.RequestMapping; |
||||
|
import org.springframework.web.bind.annotation.RestController; |
||||
|
|
||||
|
@RestController |
||||
|
@RequestMapping("/api/job/manager/executor") |
||||
|
public class ExecutorCallbackWebController { |
||||
|
@Autowired private TaskLogService taskLogService; |
||||
|
|
||||
|
@PostMapping("callback") |
||||
|
public void callback(@RequestBody TaskLog taskLog){ |
||||
|
taskLogService.callback(taskLog); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,28 @@ |
|||||
|
package io.sc.platform.job.manager.controller; |
||||
|
|
||||
|
import io.sc.platform.job.core.ExecutorRegistry; |
||||
|
import io.sc.platform.job.manager.jpa.entity.ExecutorRegistryEntity; |
||||
|
import io.sc.platform.job.manager.jpa.repository.ExecutorRegistryRepository; |
||||
|
import io.sc.platform.job.manager.service.ExecutorRegistryService; |
||||
|
import io.sc.platform.mvc.controller.support.RestCrudController; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.web.bind.annotation.PostMapping; |
||||
|
import org.springframework.web.bind.annotation.RequestBody; |
||||
|
import org.springframework.web.bind.annotation.RequestMapping; |
||||
|
import org.springframework.web.bind.annotation.RestController; |
||||
|
|
||||
|
@RestController |
||||
|
@RequestMapping("/api/job/manager/executor") |
||||
|
public class ExecutorRegistryWebController { |
||||
|
@Autowired private ExecutorRegistryService executorRegistryService; |
||||
|
|
||||
|
@PostMapping("registry") |
||||
|
public void registry(@RequestBody ExecutorRegistry executorRegistry){ |
||||
|
executorRegistryService.registry(executorRegistry); |
||||
|
} |
||||
|
|
||||
|
@PostMapping("unRegistry") |
||||
|
public void unRegistry(@RequestBody ExecutorRegistry executorRegistry){ |
||||
|
executorRegistryService.unRegistry(executorRegistry); |
||||
|
} |
||||
|
} |
||||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue