wyl/canal
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fixed issue #2099 , support user/passwd acl and canal admin operator

Browse Source
This commit is contained in:
agapple
2019-08-26 18:16:33 +08:00
parent e376a129ed
commit 3d4dbc9a8c
54 changed files with 8518 additions and 813 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+5 -4
View File
@@ -18,7 +18,8 @@ jtester.properties
*.rpm
client-adapter/example/
*.dat
canal-admin/canal-admin-ui/dist
canal-admin/canal-admin-ui/node
canal-admin/canal-admin-ui/node_modules
conf/
canal-admin/canal-admin-ui/dist/
canal-admin/canal-admin-ui/node/
canal-admin/canal-admin-ui/node_modules/
canal-admin/canal-admin-server/src/main/resources/conf/
canal-admin/canal-admin-server/pom.xml
+63
View File
@@ -12,6 +12,12 @@
<artifactId>canal-admin-server</artifactId>
<dependencies>
<dependency>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal.protocol</artifactId>
<version>${project.version}</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
@@ -134,6 +140,63 @@
</resources>
</configuration>
</execution>
<execution>
<id>copy canal default conf</id>
<phase>generate-resources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>src/main/resources/conf</outputDirectory>
<overwrite>true</overwrite>
<resources>
<resource>
<directory>${project.parent.basedir}/../deployer/src/main/resources</directory>
<includes>
<include>canal.properties</include>
</includes>
</resource>
</resources>
</configuration>
</execution>
<execution>
<id>copy instance default conf</id>
<phase>generate-resources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>src/main/resources/conf</outputDirectory>
<overwrite>true</overwrite>
<resources>
<resource>
<directory>${project.parent.basedir}/../deployer/src/main/resources/example</directory>
<includes>
<include>instance.properties</include>
</includes>
</resource>
</resources>
</configuration>
</execution>
<execution>
<id>copy client default conf</id>
<phase>generate-resources</phase>
<goals>
<goal>copy-resources</goal>
</goals>
<configuration>
<outputDirectory>src/main/resources/conf</outputDirectory>
<overwrite>true</overwrite>
<resources>
<resource>
<directory>${project.parent.basedir}/../client-adapter/launcher/src/main/resources</directory>
<includes>
<include>application.yml</include>
</includes>
</resource>
</resources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/common/exception/ServiceException.java
+16 -14
View File
@@ -25,23 +25,25 @@ import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseStatus(value = HttpStatus.SERVICE_UNAVAILABLE)
public class ServiceException extends RuntimeException {
public ServiceException() {
}
private static final long serialVersionUID = -663217666968123330L;
public ServiceException(String message) {
super(message);
}
public ServiceException(){
}
public ServiceException(String message, Throwable cause) {
super(message, cause);
}
public ServiceException(String message){
super(message);
}
public ServiceException(Throwable cause) {
super(cause);
}
public ServiceException(String message, Throwable cause){
super(message, cause);
}
public ServiceException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
public ServiceException(Throwable cause){
super(cause);
}
public ServiceException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace){
super(message, cause, enableSuppression, writableStackTrace);
}
}
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/connector/AdminConnector.java
+124
View File
@@ -0,0 +1,124 @@
package com.alibaba.otter.canal.admin.connector;
import com.alibaba.otter.canal.admin.common.exception.ServiceException;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
/**
* canal数据操作客户端
*
* @author zebin.xuzb @ 2012-6-19
* @author jianghang
* @version 1.0.0
*/
public interface AdminConnector {
/**
* 链接对应的canal server
*
* @throws CanalClientException
*/
void connect() throws ServiceException;
/**
* 释放链接
*
* @throws CanalClientException
*/
void disconnect() throws ServiceException;
/**
* 获取Canal Server状态
*
* @return 状态代码
*/
boolean check();
/**
* 启动Canal Server
*
* @return 是否成功
*/
boolean start();
/**
* 停止Canal Server
*
* @return 是否成功
*/
boolean stop();
/**
* 重启Canal Server
*
* @return 是否成功
*/
boolean restart();
/**
* 获取所有当前节点下运行中的实例
*
* @return 实例信息
*/
String getRunningInstances();
/**
* 通过实例名检查
*
* @param destination
* @return
*/
boolean checkInstance(String destination);
/**
* 通过实例名启动实例
*
* @param destination 实例名
* @return 是否成功
*/
boolean startInstance(String destination);
/**
* 通过实例名关闭实例
*
* @param destination 实例名
* @return 是否成功
*/
boolean stopInstance(String destination);
/**
* 通过实例名重启实例
*
* @param destination 实例名
* @return 是否成功
*/
boolean restartInstance(String destination);
/**
* 获取Canal Server日志列表
*
* @return 日志信息
*/
String listCanalLog();
/**
* 获取Canal Server日志
*
* @return 日志信息
*/
String canalLog(int lines);
/**
* 获取Instance的机器日志列表
*
* @param destination
*/
String listInstanceLog(String destination);
/**
* 通过实例名获取实例日志
*
* @return 日志信息
*/
String instanceLog(String destination, String fileName, int lines);
}
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnector.java
+318
View File
@@ -0,0 +1,318 @@
package com.alibaba.otter.canal.admin.connector;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.Channel;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.SocketChannel;
import java.nio.channels.WritableByteChannel;
import java.security.NoSuchAlgorithmException;
import org.apache.commons.lang.BooleanUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.admin.common.exception.ServiceException;
import com.alibaba.otter.canal.protocol.AdminPacket.Ack;
import com.alibaba.otter.canal.protocol.AdminPacket.ClientAuth;
import com.alibaba.otter.canal.protocol.AdminPacket.Handshake;
import com.alibaba.otter.canal.protocol.AdminPacket.InstanceAdmin;
import com.alibaba.otter.canal.protocol.AdminPacket.LogAdmin;
import com.alibaba.otter.canal.protocol.AdminPacket.Packet;
import com.alibaba.otter.canal.protocol.AdminPacket.PacketType;
import com.alibaba.otter.canal.protocol.AdminPacket.ServerAdmin;
import com.alibaba.otter.canal.protocol.SecurityUtil;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.google.protobuf.ByteString;
/**
* 基于netty实现的admin控制
*
* @author agapple 2019年8月26日 上午10:23:44
* @since 1.1.4
*/
public class SimpleAdminConnector implements AdminConnector {
private static final Logger logger = LoggerFactory.getLogger(SimpleAdminConnector.class);
private String user;
private String passwd;
private SocketAddress address;
private int soTimeout = 60000; // milliseconds
private int idleTimeout = 60 * 60 * 1000; // client和server之间的空闲链接超时的时间,默认为1小时
private final ByteBuffer readHeader = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
private final ByteBuffer writeHeader = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
private SocketChannel channel;
private ReadableByteChannel readableChannel;
private WritableByteChannel writableChannel;
private volatile boolean connected = false;
public SimpleAdminConnector(String ip, int port, String user, String passwd){
this.address = new InetSocketAddress(ip, port);
this.user = user;
this.passwd = passwd;
}
@Override
public void connect() throws ServiceException {
try {
if (connected) {
return;
}
channel = SocketChannel.open();
channel.socket().setSoTimeout(soTimeout);
channel.connect(address);
readableChannel = Channels.newChannel(channel.socket().getInputStream());
writableChannel = Channels.newChannel(channel.socket().getOutputStream());
Packet p = Packet.parseFrom(readNextPacket());
if (p.getVersion() != 1) {
throw new CanalClientException("unsupported version at this client.");
}
if (p.getType() != PacketType.HANDSHAKE) {
throw new CanalClientException("expect handshake but found other type.");
}
Handshake handshake = Handshake.parseFrom(p.getBody());
ByteString seed = handshake.getSeeds(); // seed for auth
String newPasswd = passwd;
if (passwd != null) {
// encode passwd
newPasswd = SecurityUtil.byte2HexStr(SecurityUtil.scramble411(passwd.getBytes(), seed.toByteArray()));
}
ClientAuth ca = ClientAuth.newBuilder()
.setUsername(user != null ? user : "")
.setPassword(ByteString.copyFromUtf8(newPasswd != null ? newPasswd : ""))
.setNetReadTimeout(idleTimeout)
.setNetWriteTimeout(idleTimeout)
.build();
writeWithHeader(Packet.newBuilder()
.setType(PacketType.CLIENTAUTHENTICATION)
.setBody(ca.toByteString())
.build()
.toByteArray());
//
Packet ack = Packet.parseFrom(readNextPacket());
if (ack.getType() != PacketType.ACK) {
throw new CanalClientException("unexpected packet type when ack is expected");
}
Ack ackBody = Ack.parseFrom(ack.getBody());
if (ackBody.getCode() > 0) {
throw new ServiceException("something goes wrong when doing authentication: " + ackBody.getMessage());
}
connected = true;
} catch (IOException e) {
throw new ServiceException(e);
} catch (NoSuchAlgorithmException e) {
throw new ServiceException(e);
}
}
@Override
public void disconnect() throws ServiceException {
if (!connected) {
return;
}
connected = false;
if (readableChannel != null) {
quietlyClose(readableChannel);
readableChannel = null;
}
if (writableChannel != null) {
quietlyClose(writableChannel);
writableChannel = null;
}
if (channel != null) {
quietlyClose(channel);
channel = null;
}
}
@Override
public boolean check() {
return BooleanUtils.toBoolean(Integer.valueOf(doServerAdmin("check")));
}
@Override
public boolean start() {
return BooleanUtils.toBoolean(Integer.valueOf(doServerAdmin("start")));
}
@Override
public boolean stop() {
return BooleanUtils.toBoolean(Integer.valueOf(doServerAdmin("stop")));
}
@Override
public boolean restart() {
return BooleanUtils.toBoolean(Integer.valueOf(doServerAdmin("restart")));
}
@Override
public String getRunningInstances() {
return doServerAdmin("list");
}
@Override
public boolean checkInstance(String destination) {
return BooleanUtils.toBoolean(Integer.valueOf(doInstanceAdmin(destination, "check")));
}
@Override
public boolean startInstance(String destination) {
return BooleanUtils.toBoolean(Integer.valueOf(doInstanceAdmin(destination, "start")));
}
@Override
public boolean stopInstance(String destination) {
return BooleanUtils.toBoolean(Integer.valueOf(doInstanceAdmin(destination, "stop")));
}
@Override
public boolean restartInstance(String destination) {
return BooleanUtils.toBoolean(Integer.valueOf(doInstanceAdmin(destination, "restart")));
}
@Override
public String listCanalLog() {
return doLogAdmin("server", "list", null, null, 0);
}
@Override
public String canalLog(int lines) {
return doLogAdmin("server", "file", null, null, lines);
}
@Override
public String listInstanceLog(String destination) {
return doLogAdmin("instance", "list", destination, null, 0);
}
@Override
public String instanceLog(String destination, String fileName, int lines) {
return doLogAdmin("instance", "file", destination, fileName, lines);
}
// ==================== helper method ====================
private String doServerAdmin(String action) {
try {
writeWithHeader(Packet.newBuilder()
.setType(PacketType.SERVER)
.setBody(ServerAdmin.newBuilder().setAction(action).build().toByteString())
.build()
.toByteArray());
Packet p = Packet.parseFrom(readNextPacket());
Ack ack = Ack.parseFrom(p.getBody());
if (ack.getCode() > 0) {
throw new ServiceException("failed to subscribe with reason: " + ack.getMessage());
}
return ack.getMessage();
} catch (IOException e) {
throw new ServiceException(e);
}
}
private String doInstanceAdmin(String destination, String action) {
try {
writeWithHeader(Packet.newBuilder()
.setType(PacketType.INSTANCE)
.setBody(InstanceAdmin.newBuilder()
.setDestination(destination)
.setAction(action)
.build()
.toByteString())
.build()
.toByteArray());
Packet p = Packet.parseFrom(readNextPacket());
Ack ack = Ack.parseFrom(p.getBody());
if (ack.getCode() > 0) {
throw new ServiceException("failed to subscribe with reason: " + ack.getMessage());
}
return ack.getMessage();
} catch (IOException e) {
throw new ServiceException(e);
}
}
private String doLogAdmin(String type, String action, String destination, String file, int count) {
try {
writeWithHeader(Packet.newBuilder()
.setType(PacketType.LOG)
.setBody(LogAdmin.newBuilder()
.setType(type)
.setAction(action)
.setDestination(destination == null ? "" : destination)
.setFile(file == null ? "" : file)
.setCount(count)
.build()
.toByteString())
.build()
.toByteArray());
Packet p = Packet.parseFrom(readNextPacket());
Ack ack = Ack.parseFrom(p.getBody());
if (ack.getCode() > 0) {
throw new ServiceException("failed to subscribe with reason: " + ack.getMessage());
}
return ack.getMessage();
} catch (IOException e) {
throw new ServiceException(e);
}
}
private void writeWithHeader(byte[] body) throws IOException {
writeWithHeader(writableChannel, body);
}
private byte[] readNextPacket() throws IOException {
return readNextPacket(readableChannel);
}
private void writeWithHeader(WritableByteChannel channel, byte[] body) throws IOException {
writeHeader.clear();
writeHeader.putInt(body.length);
writeHeader.flip();
channel.write(writeHeader);
channel.write(ByteBuffer.wrap(body));
}
private byte[] readNextPacket(ReadableByteChannel channel) throws IOException {
readHeader.clear();
read(channel, readHeader);
int bodyLen = readHeader.getInt(0);
ByteBuffer bodyBuf = ByteBuffer.allocate(bodyLen).order(ByteOrder.BIG_ENDIAN);
read(channel, bodyBuf);
return bodyBuf.array();
}
private void read(ReadableByteChannel channel, ByteBuffer buffer) throws IOException {
while (buffer.hasRemaining()) {
int r = channel.read(buffer);
if (r == -1) {
throw new IOException("end of stream when reading header");
}
}
}
private void quietlyClose(Channel channel) {
try {
channel.close();
} catch (IOException e) {
logger.warn("exception on closing channel:{} \n {}", channel, e);
}
}
}
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/connector/SimpleAdminConnectors.java
+31
View File
@@ -0,0 +1,31 @@
package com.alibaba.otter.canal.admin.connector;
import java.util.function.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SimpleAdminConnectors {
private static final Logger logger = LoggerFactory.getLogger(SimpleAdminConnectors.class);
private static String defaultUser = "admin";
private static String defaultPasswd = "admin";
public static <R> R execute(String ip, int port, Function<AdminConnector, R> function) {
return execute(ip, port, defaultUser, defaultPasswd, function);
}
public static <R> R execute(String ip, int port, String user, String passwd, Function<AdminConnector, R> function) {
SimpleAdminConnector connector = new SimpleAdminConnector(ip, port, user, passwd);
try {
connector.connect();
return function.apply(connector);
} catch (Exception e) {
logger.error(e.getMessage());
} finally {
connector.disconnect();
}
return null;
}
}
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/controller/CanalInstanceController.java
+20 -1
View File
@@ -4,7 +4,14 @@ import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.PutMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.alibaba.otter.canal.admin.model.BaseModel;
import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
@@ -49,6 +56,18 @@ public class CanalInstanceController {
return BaseModel.getInstance("success");
}
/**
* 实例详情信息
*
* @param id 实例配置id
* @param env 环境变量
* @return 实例信息
*/
@GetMapping(value = "/instance")
public BaseModel<CanalInstanceConfig> config(@PathVariable Long id, @PathVariable String env) {
return BaseModel.getInstance(canalInstanceConfigService.detail(id));
}
/**
* 实例详情信息
*
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/jmx/JMXConnection.java
-164
View File
@@ -1,164 +0,0 @@
package com.alibaba.otter.canal.admin.jmx;
import com.alibaba.otter.canal.admin.common.exception.ServiceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.management.*;
import javax.management.remote.JMXConnector;
import javax.management.remote.JMXConnectorFactory;
import javax.management.remote.JMXServiceURL;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.SocketTimeoutException;
import java.util.concurrent.*;
import java.util.function.Function;
/**
* Canal Server JMX 控制层
*
* @author rewerma 2019-07-13 下午05:12:16
* @version 1.0.0
*/
public class JMXConnection {
private static final Logger logger = LoggerFactory.getLogger(JMXConnection.class);
private String ip;
private Integer port;
private JMXConnector jmxc;
private CanalServerMXBean canalServerMXBean;
public JMXConnection(String ip, Integer port){
this.ip = ip;
this.port = port;
}
/**
* 执行相关操作
*
* @param ip jmx ip
* @param port jmx port
* @param function 执行方法
* @param <R> 返回泛型
* @return 执行结果
*/
public static <R> R execute(String ip, int port, Function<CanalServerMXBean, R> function) {
JMXConnection jmxConnection = new JMXConnection(ip, port);
try {
CanalServerMXBean canalServerMXBean = jmxConnection.getCanalServerMXBean();
return function.apply(canalServerMXBean);
} catch (Exception e) {
logger.error(e.getMessage());
} finally {
jmxConnection.close();
}
return null;
}
/**
* 连接远程jmx
*/
public void connect() {
try {
JMXServiceURL url = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://" + ip + ":" + port + "/jmxrmi");
// jmxc = JMXConnectorFactory.connect(url, null);
jmxc = connectWithTimeout(url, 3, TimeUnit.SECONDS);
MBeanServerConnection mBeanServerConnection = jmxc.getMBeanServerConnection();
ObjectName name = new ObjectName("CanalServerAgent:type=CanalServerStatus");
mBeanServerConnection.addNotificationListener(name, (notification, handback) -> {
}, null, null);
canalServerMXBean = MBeanServerInvocationHandler
.newProxyInstance(mBeanServerConnection, name, CanalServerMXBean.class, false);
} catch (Exception e) {
throw new ServiceException(e.getMessage(), e);
}
}
/**
* 带超时的jmx连接器
*
* @param url jmx url
* @param timeout 超时时间
* @param unit 超时时间单位
* @return JMXConnector
* @throws IOException 连接异常
*/
private static JMXConnector connectWithTimeout(final JMXServiceURL url, long timeout,
TimeUnit unit) throws IOException {
final BlockingQueue<Object> mailbox = new ArrayBlockingQueue<>(1);
ExecutorService executor = Executors.newSingleThreadExecutor(daemonThreadFactory);
executor.submit(() -> {
try {
JMXConnector connector = JMXConnectorFactory.connect(url);
if (!mailbox.offer(connector)) connector.close();
} catch (Throwable t) {
mailbox.offer(t);
}
});
Object result;
try {
result = mailbox.poll(timeout, unit);
if (result == null) {
if (!mailbox.offer("")) result = mailbox.take();
}
} catch (InterruptedException e) {
throw initCause(new InterruptedIOException(e.getMessage()), e);
} finally {
executor.shutdown();
}
if (result == null) throw new SocketTimeoutException("Connect timed out: " + url);
if (result instanceof JMXConnector) return (JMXConnector) result;
try {
throw (Throwable) result;
} catch (IOException | RuntimeException | Error e) {
throw e;
} catch (Throwable e) {
// In principle this can't happen but we wrap it anyway
throw new IOException(e.toString(), e);
}
}
private static <T extends Throwable> T initCause(T wrapper, Throwable wrapped) {
wrapper.initCause(wrapped);
return wrapper;
}
private static class DaemonThreadFactory implements ThreadFactory {
public Thread newThread(Runnable r) {
Thread t = Executors.defaultThreadFactory().newThread(r);
t.setDaemon(true);
return t;
}
}
private static final ThreadFactory daemonThreadFactory = new DaemonThreadFactory();
/**
* 获取MBean
*
* @return canalServerMXBean
*/
public CanalServerMXBean getCanalServerMXBean() {
if (jmxc == null) {
connect();
}
return canalServerMXBean;
}
/**
* 关闭jmx连接
*/
public void close() {
try {
if (jmxc != null) {
jmxc.close();
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally {
jmxc = null;
}
}
}
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/model/Model.java
+4 -2
View File
@@ -2,12 +2,14 @@ package com.alibaba.otter.canal.admin.model;
import io.ebean.Ebean;
import io.ebean.EbeanServer;
import org.apache.commons.beanutils.PropertyUtils;
import java.lang.reflect.Field;
import javax.persistence.Id;
import javax.persistence.MappedSuperclass;
import javax.persistence.OptimisticLockException;
import java.lang.reflect.Field;
import org.apache.commons.beanutils.PropertyUtils;
/**
* EBean Model扩展类
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/service/impl/CanalConfigServiceImpl.java
+71 -3
View File
@@ -1,5 +1,13 @@
package com.alibaba.otter.canal.admin.service.impl;
import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import com.alibaba.otter.canal.admin.model.CanalConfig;
@@ -14,15 +22,75 @@ import com.alibaba.otter.canal.admin.service.CanalConfigService;
@Service
public class CanalConfigServiceImpl implements CanalConfigService {
private Logger logger = LoggerFactory.getLogger(CanalConfigServiceImpl.class);
private static final String CANAL_GLOBAL_CONFIG = "canal.properties";
private static final String CANAL_ADAPTER_CONFIG = "application.yml";
public CanalConfig getCanalConfig() {
return CanalConfig.find.byId(1L);
long id = 1L;
CanalConfig config = CanalConfig.find.byId(id);
if (config == null) {
String context = loadDefaultConf(CANAL_GLOBAL_CONFIG);
if (context == null) {
return null;
}
config = new CanalConfig();
config.setId(id);
config.setName(CANAL_GLOBAL_CONFIG);
config.setModifiedTime(new Date());
config.setContent(context);
return config;
}
return config;
}
public CanalConfig getAdapterConfig() {
return CanalConfig.find.byId(2L);
long id = 2L;
CanalConfig config = CanalConfig.find.byId(id);
if (config == null) {
String context = loadDefaultConf(CANAL_ADAPTER_CONFIG);
if (context == null) {
return null;
}
config = new CanalConfig();
config.setId(id);
config.setName(CANAL_ADAPTER_CONFIG);
config.setModifiedTime(new Date());
config.setContent(context);
return config;
}
return config;
}
public void updateContent(CanalConfig canalConfig) {
canalConfig.update("content");
try {
canalConfig.insert();
} catch (Throwable e) {
canalConfig.update();
}
}
private String loadDefaultConf(String confFileName) {
InputStream input = null;
try {
input = Thread.currentThread().getContextClassLoader().getResourceAsStream("conf/" + confFileName);
if (input == null) {
return null;
}
return StringUtils.join(IOUtils.readLines(input), "\n");
} catch (IOException e) {
logger.error("find " + confFileName + " is error!", e);
return null;
} finally {
if (input != null) {
IOUtils.closeQuietly(input);
}
}
}
}
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/service/impl/CanalInstanceServiceImpl.java
+13 -12
View File
@@ -1,5 +1,7 @@
package com.alibaba.otter.canal.admin.service.impl;
import io.ebean.Query;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -7,14 +9,12 @@ import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Service;
import com.alibaba.otter.canal.admin.jmx.CanalServerMXBean;
import com.alibaba.otter.canal.admin.jmx.JMXConnection;
import com.alibaba.otter.canal.admin.connector.AdminConnector;
import com.alibaba.otter.canal.admin.connector.SimpleAdminConnectors;
import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
import com.alibaba.otter.canal.admin.model.NodeServer;
import com.alibaba.otter.canal.admin.service.CanalInstanceService;
import io.ebean.Query;
/**
* Canal实例配置信息业务层
*
@@ -39,8 +39,9 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
// check all canal instances running status
List<NodeServer> nodeServers = NodeServer.find.query().findList();
for (NodeServer nodeServer : nodeServers) {
String runningInstances = JMXConnection
.execute(nodeServer.getIp(), nodeServer.getPort(), CanalServerMXBean::getRunningInstances);
String runningInstances = SimpleAdminConnectors.execute(nodeServer.getIp(),
nodeServer.getPort(),
AdminConnector::getRunningInstances);
if (runningInstances == null) {
continue;
}
@@ -90,9 +91,9 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
return result;
}
String log = JMXConnection.execute(nodeServer.getIp(),
String log = SimpleAdminConnectors.execute(nodeServer.getIp(),
nodeServer.getPort(),
canalServerMXBean -> canalServerMXBean.instanceLog(canalInstanceConfig.getName()));
adminConnector -> adminConnector.instanceLog(canalInstanceConfig.getName(), null, 100));
result.put("instance", canalInstanceConfig.getName());
result.put("log", log);
@@ -122,13 +123,13 @@ public class CanalInstanceServiceImpl implements CanalInstanceService {
}
Boolean resutl = null;
if ("start".equals(option)) {
resutl = JMXConnection.execute(nodeServer.getIp(),
resutl = SimpleAdminConnectors.execute(nodeServer.getIp(),
nodeServer.getPort(),
canalServerMXBean -> canalServerMXBean.startInstance(canalInstanceConfig.getName()));
adminConnector -> adminConnector.startInstance(canalInstanceConfig.getName()));
} else if ("stop".equals(option)) {
resutl = JMXConnection.execute(nodeServer.getIp(),
resutl = SimpleAdminConnectors.execute(nodeServer.getIp(),
nodeServer.getPort(),
canalServerMXBean -> canalServerMXBean.stopInstance(canalInstanceConfig.getName()));
adminConnector -> adminConnector.stopInstance(canalInstanceConfig.getName()));
} else {
return false;
}
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/service/impl/NodeServerServiceImpl.java
+18 -32
View File
@@ -1,5 +1,7 @@
package com.alibaba.otter.canal.admin.service.impl;
import io.ebean.Query;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
@@ -8,18 +10,14 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import com.alibaba.otter.canal.admin.common.exception.ServiceException;
import com.alibaba.otter.canal.admin.jmx.CanalServerMXBean;
import com.alibaba.otter.canal.admin.jmx.JMXConnection;
import com.alibaba.otter.canal.admin.connector.AdminConnector;
import com.alibaba.otter.canal.admin.connector.SimpleAdminConnectors;
import com.alibaba.otter.canal.admin.model.NodeServer;
import com.alibaba.otter.canal.admin.service.NodeServerService;
import io.ebean.Query;
/**
* 节点信息业务层
*
@@ -29,8 +27,6 @@ import io.ebean.Query;
@Service
public class NodeServerServiceImpl implements NodeServerService {
private static final Logger logger = LoggerFactory.getLogger(NodeServerServiceImpl.class);
public void save(NodeServer nodeServer) {
int cnt = NodeServer.find.query()
.where()
@@ -90,18 +86,9 @@ public class NodeServerServiceImpl implements NodeServerService {
// get all nodes status
for (NodeServer ns : nodeServers) {
futures.add(executorService.submit(() -> {
int status = -1;
JMXConnection jmxConnection = new JMXConnection(ns.getIp(), ns.getPort());
try {
CanalServerMXBean canalServerMXBean = jmxConnection.getCanalServerMXBean();
status = canalServerMXBean.getStatus();
} catch (Exception e) {
logger.warn(e.getMessage());
} finally {
jmxConnection.close();
}
ns.setStatus(status);
return status != -1;
boolean status = SimpleAdminConnectors.execute(ns.getIp(), ns.getPort(), AdminConnector::check);
ns.setStatus(status ? 1 : 0);
return !status;
}));
}
futures.forEach(f -> {
@@ -118,11 +105,8 @@ public class NodeServerServiceImpl implements NodeServerService {
}
public int remoteNodeStatus(String ip, Integer port) {
Integer resutl = JMXConnection.execute(ip, port, CanalServerMXBean::getStatus);
if (resutl == null) {
resutl = -1;
}
return resutl;
boolean result = SimpleAdminConnectors.execute(ip, port, AdminConnector::check);
return result ? 1 : 0;
}
public String remoteCanalLog(Long id) {
@@ -130,7 +114,9 @@ public class NodeServerServiceImpl implements NodeServerService {
if (nodeServer == null) {
return "";
}
return JMXConnection.execute(nodeServer.getIp(), nodeServer.getPort(), CanalServerMXBean::canalLog);
return SimpleAdminConnectors.execute(nodeServer.getIp(),
nodeServer.getPort(),
adminConnector -> adminConnector.canalLog(100));
}
public boolean remoteOperation(Long id, String option) {
@@ -138,18 +124,18 @@ public class NodeServerServiceImpl implements NodeServerService {
if (nodeServer == null) {
return false;
}
Boolean resutl = null;
Boolean result = null;
if ("start".equals(option)) {
resutl = JMXConnection.execute(nodeServer.getIp(), nodeServer.getPort(), CanalServerMXBean::start);
result = SimpleAdminConnectors.execute(nodeServer.getIp(), nodeServer.getPort(), AdminConnector::start);
} else if ("stop".equals(option)) {
resutl = JMXConnection.execute(nodeServer.getIp(), nodeServer.getPort(), CanalServerMXBean::stop);
result = SimpleAdminConnectors.execute(nodeServer.getIp(), nodeServer.getPort(), AdminConnector::stop);
} else {
return false;
}
if (resutl == null) {
resutl = false;
if (result == null) {
result = false;
}
return resutl;
return result;
}
}
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/service/impl/UserServiceImpl.java
+31 -7
View File
@@ -1,10 +1,14 @@
package com.alibaba.otter.canal.admin.service.impl;
import java.security.NoSuchAlgorithmException;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Service;
import com.alibaba.otter.canal.admin.common.exception.ServiceException;
import com.alibaba.otter.canal.admin.model.User;
import com.alibaba.otter.canal.admin.service.UserService;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Service;
import com.alibaba.otter.canal.protocol.SecurityUtil;
/**
* 用户信息业务层
@@ -15,11 +19,22 @@ import org.springframework.stereotype.Service;
@Service
public class UserServiceImpl implements UserService {
private static byte[] seeds = "canal is best!".getBytes();
public User find4Login(String username, String password) {
if (StringUtils.isEmpty(username) || StringUtils.isEmpty(password)) {
return null;
}
User user = User.find.query().where().eq("username", username).eq("password", password).findOne();
User user = User.find.query().where().eq("username", username).findOne();
try {
byte[] pass = SecurityUtil.scramble411(password.getBytes(), seeds);
if (!SecurityUtil.scrambleServerAuth(pass, SecurityUtil.hexStr2Bytes(user.getPassword()), seeds)) {
throw new ServiceException("user:" + user.getName() + " passwd incorrect!");
}
} catch (NoSuchAlgorithmException e) {
throw new ServiceException("user:" + user.getName() + " auth failed!");
}
if (user != null) {
user.setPassword("");
}
@@ -27,14 +42,23 @@ public class UserServiceImpl implements UserService {
}
public void update(User user) {
User userTmp = User.find.byId(1L);
User userTmp = User.find.query().where().eq("username", user.getUsername()).findOne();
if (userTmp == null) {
throw new ServiceException();
}
if (!userTmp.getPassword().equals(user.getOldPassword())) {
throw new ServiceException("错误的旧密码");
try {
byte[] pass = SecurityUtil.scramble411(user.getOldPassword().getBytes(), seeds);
if (!SecurityUtil.scrambleServerAuth(pass, SecurityUtil.hexStr2Bytes(userTmp.getPassword()), seeds)) {
throw new ServiceException("old passwd is unmatch");
}
user.setId(userTmp.getId());
user.setPassword(SecurityUtil.scrambleGenPass(user.getPassword().getBytes()));
} catch (NoSuchAlgorithmException e) {
throw new ServiceException("passwd process failed");
}
user.setId(1L);
user.update("username", "nn:password");
}
}
canal-admin/canal-admin-server/src/main/resources/canal_manager.sql
+85
View File
@@ -0,0 +1,85 @@
CREATE DATABASE /*!32312 IF NOT EXISTS*/ `canal_manager` /*!40100 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin */;
USE `canal_manager`;
SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;
-- ----------------------------
-- Table structure for canal_adapter_config
-- ----------------------------
DROP TABLE IF EXISTS `canal_adapter_config`;
CREATE TABLE `canal_adapter_config` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`category` varchar(45) NOT NULL,
`name` varchar(45) NOT NULL,
`content` text NOT NULL,
`modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4;
-- ----------------------------
-- Table structure for canal_config
-- ----------------------------
DROP TABLE IF EXISTS `canal_config`;
CREATE TABLE `canal_config` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`name` varchar(45) NOT NULL,
`content` text NOT NULL,
`modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `name_UNIQUE` (`name`)
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4;
-- ----------------------------
-- Table structure for canal_instance_config
-- ----------------------------
DROP TABLE IF EXISTS `canal_instance_config`;
CREATE TABLE `canal_instance_config` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`name` varchar(45) NOT NULL,
`content` text NOT NULL,
`modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `name_UNIQUE` (`name`)
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4;
-- ----------------------------
-- Table structure for canal_node_server
-- ----------------------------
DROP TABLE IF EXISTS `canal_node_server`;
CREATE TABLE `canal_node_server` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`name` varchar(63) NOT NULL,
`ip` varchar(63) NOT NULL,
`port` int(11) DEFAULT NULL,
`port2` int(11) DEFAULT NULL,
`status` int(11) NOT NULL,
`modified_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;
-- ----------------------------
-- Table structure for canal_user
-- ----------------------------
DROP TABLE IF EXISTS `canal_user`;
CREATE TABLE `canal_user` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`username` varchar(31) NOT NULL,
`password` varchar(128) NOT NULL,
`name` varchar(31) NOT NULL,
`roles` varchar(31) NOT NULL,
`introduction` varchar(255) DEFAULT NULL,
`avatar` varchar(255) DEFAULT NULL,
`creation_date` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4;
-- ----------------------------
-- Records of canal_user
-- ----------------------------
BEGIN;
INSERT INTO `canal_user` VALUES (1, 'admin', '6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9', 'Canal Manager', 'admin', NULL, NULL, '2019-07-14 00:05:28');
COMMIT;
SET FOREIGN_KEY_CHECKS = 1;
canal-admin/canal-admin-server/src/main/resources/public/index.html
+1 -1
View File
@@ -1 +1 @@
<!DOCTYPE html><html><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=viewport content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><link rel=icon href=/favicon.ico><title>Canal Admin</title><link href=/static/css/chunk-elementUI.18b11d0e.css rel=stylesheet><link href=/static/css/chunk-libs.5cf311f0.css rel=stylesheet><link href=/static/css/app.bb951cb3.css rel=stylesheet></head><body><noscript><strong>We're sorry but Canal Admin doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id=app></div><script src=/static/js/chunk-elementUI.667f4c87.js></script><script src=/static/js/chunk-libs.c04beefc.js></script><script>(function(e){function n(n){for(var r,c,f=n[0],a=n[1],i=n[2],h=0,l=[];h<f.length;h++)c=f[h],u[c]&&l.push(u[c][0]),u[c]=0;for(r in a)Object.prototype.hasOwnProperty.call(a,r)&&(e[r]=a[r]);d&&d(n);while(l.length)l.shift()();return o.push.apply(o,i||[]),t()}function t(){for(var e,n=0;n<o.length;n++){for(var t=o[n],r=!0,c=1;c<t.length;c++){var f=t[c];0!==u[f]&&(r=!1)}r&&(o.splice(n--,1),e=a(a.s=t[0]))}return e}var r={},c={runtime:0},u={runtime:0},o=[];function f(e){return a.p+"static/js/"+({}[e]||e)+"."+{"chunk-101fc062":"372a5ca5","chunk-137eaf06":"539d27c7","chunk-238a81e9":"da844a27","chunk-37c49cbf":"92ebe0ae","chunk-3fcdf643":"4b7133b8","chunk-1ef583f0":"c799c803","chunk-5560c396":"5eb91155","chunk-6f6d6145":"385d3c06","chunk-4f09fed2":"d107437b","chunk-69386cf0":"bdbe3f0c","chunk-7ec889b7":"192dc04c","chunk-e1a839e4":"3175ead5"}[e]+".js"}function a(n){if(r[n])return r[n].exports;var t=r[n]={i:n,l:!1,exports:{}};return e[n].call(t.exports,t,t.exports,a),t.l=!0,t.exports}a.e=function(e){var n=[],t={"chunk-101fc062":1,"chunk-238a81e9":1,"chunk-37c49cbf":1,"chunk-1ef583f0":1,"chunk-5560c396":1,"chunk-6f6d6145":1,"chunk-4f09fed2":1,"chunk-69386cf0":1,"chunk-7ec889b7":1};c[e]?n.push(c[e]):0!==c[e]&&t[e]&&n.push(c[e]=new Promise(function(n,t){for(var r="static/css/"+({}[e]||e)+"."+{"chunk-101fc062":"fad9926f","chunk-137eaf06":"31d6cfe0","chunk-238a81e9":"e8e2beee","chunk-37c49cbf":"efc21a9c","chunk-3fcdf643":"31d6cfe0","chunk-1ef583f0":"ec3a43f5","chunk-5560c396":"b660603c","chunk-6f6d6145":"bf63041a","chunk-4f09fed2":"70ec0b86","chunk-69386cf0":"741ff14e","chunk-7ec889b7":"c0585512","chunk-e1a839e4":"31d6cfe0"}[e]+".css",u=a.p+r,o=document.getElementsByTagName("link"),f=0;f<o.length;f++){var i=o[f],h=i.getAttribute("data-href")||i.getAttribute("href");if("stylesheet"===i.rel&&(h===r||h===u))return n()}var l=document.getElementsByTagName("style");for(f=0;f<l.length;f++){i=l[f],h=i.getAttribute("data-href");if(h===r||h===u)return n()}var d=document.createElement("link");d.rel="stylesheet",d.type="text/css",d.onload=n,d.onerror=function(n){var r=n&&n.target&&n.target.src||u,o=new Error("Loading CSS chunk "+e+" failed.\n("+r+")");o.code="CSS_CHUNK_LOAD_FAILED",o.request=r,delete c[e],d.parentNode.removeChild(d),t(o)},d.href=u;var s=document.getElementsByTagName("head")[0];s.appendChild(d)}).then(function(){c[e]=0}));var r=u[e];if(0!==r)if(r)n.push(r[2]);else{var o=new Promise(function(n,t){r=u[e]=[n,t]});n.push(r[2]=o);var i,h=document.createElement("script");h.charset="utf-8",h.timeout=120,a.nc&&h.setAttribute("nonce",a.nc),h.src=f(e),i=function(n){h.onerror=h.onload=null,clearTimeout(l);var t=u[e];if(0!==t){if(t){var r=n&&("load"===n.type?"missing":n.type),c=n&&n.target&&n.target.src,o=new Error("Loading chunk "+e+" failed.\n("+r+": "+c+")");o.type=r,o.request=c,t[1](o)}u[e]=void 0}};var l=setTimeout(function(){i({type:"timeout",target:h})},12e4);h.onerror=h.onload=i,document.head.appendChild(h)}return Promise.all(n)},a.m=e,a.c=r,a.d=function(e,n,t){a.o(e,n)||Object.defineProperty(e,n,{enumerable:!0,get:t})},a.r=function(e){"undefined"!==typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},a.t=function(e,n){if(1&n&&(e=a(e)),8&n)return e;if(4&n&&"object"===typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(a.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&n&&"string"!=typeof e)for(var r in e)a.d(t,r,function(n){return e[n]}.bind(null,r));return t},a.n=function(e){var n=e&&e.__esModule?function(){return e["default"]}:function(){return e};return a.d(n,"a",n),n},a.o=function(e,n){return Object.prototype.hasOwnProperty.call(e,n)},a.p="/",a.oe=function(e){throw console.error(e),e};var i=window["webpackJsonp"]=window["webpackJsonp"]||[],h=i.push.bind(i);i.push=n,i=i.slice();for(var l=0;l<i.length;l++)n(i[l]);var d=h;t()})([]);</script><script src=/static/js/app.38307ef0.js></script></body></html>
<!DOCTYPE html><html><head><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge,chrome=1"><meta name=viewport content="width=device-width,initial-scale=1,maximum-scale=1,user-scalable=no"><link rel=icon href=/favicon.ico><title>Canal Admin</title><link href=/static/css/chunk-elementUI.18b11d0e.css rel=stylesheet><link href=/static/css/chunk-libs.5cf311f0.css rel=stylesheet><link href=/static/css/app.bb951cb3.css rel=stylesheet></head><body><noscript><strong>We're sorry but Canal Admin doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id=app></div><script src=/static/js/chunk-elementUI.667f4c87.js></script><script src=/static/js/chunk-libs.c04beefc.js></script><script>(function(e){function n(n){for(var r,c,f=n[0],a=n[1],i=n[2],h=0,d=[];h<f.length;h++)c=f[h],u[c]&&d.push(u[c][0]),u[c]=0;for(r in a)Object.prototype.hasOwnProperty.call(a,r)&&(e[r]=a[r]);l&&l(n);while(d.length)d.shift()();return o.push.apply(o,i||[]),t()}function t(){for(var e,n=0;n<o.length;n++){for(var t=o[n],r=!0,c=1;c<t.length;c++){var f=t[c];0!==u[f]&&(r=!1)}r&&(o.splice(n--,1),e=a(a.s=t[0]))}return e}var r={},c={runtime:0},u={runtime:0},o=[];function f(e){return a.p+"static/js/"+({}[e]||e)+"."+{"chunk-101fc062":"372a5ca5","chunk-137eaf06":"539d27c7","chunk-238a81e9":"da844a27","chunk-37c49cbf":"92ebe0ae","chunk-3fcdf643":"4b7133b8","chunk-1ef583f0":"c799c803","chunk-5560c396":"5eb91155","chunk-6f6d6145":"385d3c06","chunk-4f09fed2":"d107437b","chunk-69386cf0":"bdbe3f0c","chunk-7ec889b7":"192dc04c","chunk-e1a839e4":"7ee86dd8"}[e]+".js"}function a(n){if(r[n])return r[n].exports;var t=r[n]={i:n,l:!1,exports:{}};return e[n].call(t.exports,t,t.exports,a),t.l=!0,t.exports}a.e=function(e){var n=[],t={"chunk-101fc062":1,"chunk-238a81e9":1,"chunk-37c49cbf":1,"chunk-1ef583f0":1,"chunk-5560c396":1,"chunk-6f6d6145":1,"chunk-4f09fed2":1,"chunk-69386cf0":1,"chunk-7ec889b7":1};c[e]?n.push(c[e]):0!==c[e]&&t[e]&&n.push(c[e]=new Promise(function(n,t){for(var r="static/css/"+({}[e]||e)+"."+{"chunk-101fc062":"fad9926f","chunk-137eaf06":"31d6cfe0","chunk-238a81e9":"e8e2beee","chunk-37c49cbf":"efc21a9c","chunk-3fcdf643":"31d6cfe0","chunk-1ef583f0":"ec3a43f5","chunk-5560c396":"b660603c","chunk-6f6d6145":"bf63041a","chunk-4f09fed2":"70ec0b86","chunk-69386cf0":"741ff14e","chunk-7ec889b7":"c0585512","chunk-e1a839e4":"31d6cfe0"}[e]+".css",u=a.p+r,o=document.getElementsByTagName("link"),f=0;f<o.length;f++){var i=o[f],h=i.getAttribute("data-href")||i.getAttribute("href");if("stylesheet"===i.rel&&(h===r||h===u))return n()}var d=document.getElementsByTagName("style");for(f=0;f<d.length;f++){i=d[f],h=i.getAttribute("data-href");if(h===r||h===u)return n()}var l=document.createElement("link");l.rel="stylesheet",l.type="text/css",l.onload=n,l.onerror=function(n){var r=n&&n.target&&n.target.src||u,o=new Error("Loading CSS chunk "+e+" failed.\n("+r+")");o.code="CSS_CHUNK_LOAD_FAILED",o.request=r,delete c[e],l.parentNode.removeChild(l),t(o)},l.href=u;var s=document.getElementsByTagName("head")[0];s.appendChild(l)}).then(function(){c[e]=0}));var r=u[e];if(0!==r)if(r)n.push(r[2]);else{var o=new Promise(function(n,t){r=u[e]=[n,t]});n.push(r[2]=o);var i,h=document.createElement("script");h.charset="utf-8",h.timeout=120,a.nc&&h.setAttribute("nonce",a.nc),h.src=f(e),i=function(n){h.onerror=h.onload=null,clearTimeout(d);var t=u[e];if(0!==t){if(t){var r=n&&("load"===n.type?"missing":n.type),c=n&&n.target&&n.target.src,o=new Error("Loading chunk "+e+" failed.\n("+r+": "+c+")");o.type=r,o.request=c,t[1](o)}u[e]=void 0}};var d=setTimeout(function(){i({type:"timeout",target:h})},12e4);h.onerror=h.onload=i,document.head.appendChild(h)}return Promise.all(n)},a.m=e,a.c=r,a.d=function(e,n,t){a.o(e,n)||Object.defineProperty(e,n,{enumerable:!0,get:t})},a.r=function(e){"undefined"!==typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},a.t=function(e,n){if(1&n&&(e=a(e)),8&n)return e;if(4&n&&"object"===typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(a.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&n&&"string"!=typeof e)for(var r in e)a.d(t,r,function(n){return e[n]}.bind(null,r));return t},a.n=function(e){var n=e&&e.__esModule?function(){return e["default"]}:function(){return e};return a.d(n,"a",n),n},a.o=function(e,n){return Object.prototype.hasOwnProperty.call(e,n)},a.p="/",a.oe=function(e){throw console.error(e),e};var i=window["webpackJsonp"]=window["webpackJsonp"]||[],h=i.push.bind(i);i.push=n,i=i.slice();for(var d=0;d<i.length;d++)n(i[d]);var l=h;t()})([]);</script><script src=/static/js/app.38307ef0.js></script></body></html>
canal-admin/canal-admin-server/src/main/resources/public/static/js/chunk-e1a839e4.3175ead5.js
-1
View File
File diff suppressed because one or more lines are too long
canal-admin/canal-admin-server/src/main/resources/public/static/js/chunk-e1a839e4.7ee86dd8.js
+1
View File
File diff suppressed because one or more lines are too long
canal-admin/canal-admin-server/src/test/java/com/alibaba/otter/canal/admin/SimpleAdminConnectorTest.java
+40
View File
@@ -0,0 +1,40 @@
package com.alibaba.otter.canal.admin;
import org.junit.Test;
import com.alibaba.otter.canal.admin.connector.SimpleAdminConnector;
public class SimpleAdminConnectorTest {
@Test
public void testSimple() {
SimpleAdminConnector connector = new SimpleAdminConnector("127.0.0.1", 11110, "admin", "admin");
connector.connect();
System.out.println("check 1 : " + connector.check());
System.out.println("getRunning Before stop 1 : " + connector.getRunningInstances());
System.out.println("stop 1: " + connector.stop());
System.out.println("getRunning After stop 1 : " + connector.getRunningInstances());
System.out.println("check 1 : " + connector.check());
System.out.println("listFile 1 : " + connector.listCanalLog());
System.out.println("getFile 1 : " + connector.canalLog(10));
connector.disconnect();
connector.connect();
System.out.println("check 2 : " + connector.check());
System.out.println("get Running Before start : " + connector.getRunningInstances());
System.out.println("start 2 : " + connector.start());
System.out.println("get Running After start : " + connector.getRunningInstances());
System.out.println("check 2 : " + connector.check());
System.out.println("check after before example 2 : " + connector.checkInstance("example"));
System.out.println("stop example 2 : " + connector.stopInstance("example"));
System.out.println("check example 2 : " + connector.checkInstance("example"));
System.out.println("start example 2 : " + connector.startInstance("example"));
System.out.println("check after start example 2 : " + connector.checkInstance("example"));
System.out.println("listFile 2 : " + connector.listCanalLog());
System.out.println("getFile 2 : " + connector.canalLog(10));
System.out.println("listFile 3 : " + connector.listInstanceLog("example"));
System.out.println("getFile 3 : " + connector.instanceLog("example", "example.log", 10));
connector.disconnect();
}
}
canal-admin/canal-admin-server/src/test/java/com/alibaba/otter/canal/admin/jmx/JMXConnectionTest.java
-20
View File
@@ -1,20 +0,0 @@
package com.alibaba.otter.canal.admin.jmx;
import org.junit.Test;
public class JMXConnectionTest {
@Test
public void testSimple() {
JMXConnection jmxConnection = new JMXConnection("127.0.0.1", 11113);
try {
CanalServerMXBean canalServerMXBean = jmxConnection.getCanalServerMXBean();
System.out.println(canalServerMXBean.getStatus());
} catch (Exception e) {
e.printStackTrace();
} finally {
jmxConnection.close();
}
}
}
canal-admin/canal-admin-ui/src/views/canalServer/NodeServer.vue
+5 -5
View File
@@ -25,7 +25,7 @@
<span>{{ scope.row.ip }}</span>
</template>
</el-table-column>
<el-table-column label="JMX端口" min-width="100" align="center">
<el-table-column label="Admin 端口" min-width="100" align="center">
<template slot-scope="scope">
{{ scope.row.port }}
</template>
@@ -65,8 +65,8 @@
<el-form-item label="Server IP" prop="ip">
<el-input v-model="nodeModel.ip" />
</el-form-item>
<el-form-item label="JMX端口" prop="port">
<el-input v-model="nodeModel.port" placeholder="11113" type="number" />
<el-form-item label="Admin 端口" prop="port">
<el-input v-model="nodeModel.port" placeholder="11110" type="number" />
</el-form-item>
<el-form-item label="监控端口" prop="port2">
<el-input v-model="nodeModel.port2" placeholder="11112" type="number" />
@@ -119,13 +119,13 @@ export default {
id: undefined,
name: null,
ip: null,
port: 11113,
port: 11110,
port2: 11112
},
rules: {
name: [{ required: true, message: 'Server 名称不能为空', trigger: 'change' }],
ip: [{ required: true, message: 'Server IP不能为空', trigger: 'change' }],
port: [{ required: true, message: 'Server JMX端口不能为空', trigger: 'change' }]
port: [{ required: true, message: 'Server Admin端口不能为空', trigger: 'change' }]
},
dialogStatus: 'create'
}
canal-admin/canal_manager.sql
-106
View File
File diff suppressed because one or more lines are too long
client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java
+11 -2
View File
@@ -10,6 +10,7 @@ import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.SocketChannel;
import java.nio.channels.WritableByteChannel;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -38,6 +39,7 @@ import com.alibaba.otter.canal.protocol.CanalPacket.Sub;
import com.alibaba.otter.canal.protocol.CanalPacket.Unsub;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.SecurityUtil;
import com.alibaba.otter.canal.protocol.exception.CanalClientException;
import com.google.protobuf.ByteString;
@@ -160,9 +162,16 @@ public class SimpleCanalConnector implements CanalConnector {
Handshake handshake = Handshake.parseFrom(p.getBody());
supportedCompressions.add(handshake.getSupportedCompressions());
//
ByteString seed = handshake.getSeeds(); // seed for auth
String newPasswd = password;
if (password != null) {
// encode passwd
newPasswd = SecurityUtil.byte2HexStr(SecurityUtil.scramble411(password.getBytes(), seed.toByteArray()));
}
ClientAuth ca = ClientAuth.newBuilder()
.setUsername(username != null ? username : "")
.setPassword(ByteString.copyFromUtf8(password != null ? password : ""))
.setPassword(ByteString.copyFromUtf8(newPasswd != null ? newPasswd : ""))
.setNetReadTimeout(idleTimeout)
.setNetWriteTimeout(idleTimeout)
.build();
@@ -185,7 +194,7 @@ public class SimpleCanalConnector implements CanalConnector {
connected = true;
return new InetSocketAddress(channel.socket().getLocalAddress(), channel.socket().getLocalPort());
} catch (IOException e) {
} catch (IOException | NoSuchAlgorithmException e) {
throw new CanalClientException(e);
}
}
common/src/main/java/com/alibaba/otter/canal/common/utils/FileUtils.java
+14 -5
View File
@@ -12,20 +12,24 @@ public class FileUtils {
private static final Logger logger = LoggerFactory.getLogger(FileUtils.class);
public static String readFileFromOffset(String filename, int l, String charset) {
return readFileFromOffset(filename, l, charset, 4 * 1024 * 1024);
}
public static String readFileFromOffset(String filename, int l, String charset, int maxSize) {
RandomAccessFile rf = null;
StringBuilder res = new StringBuilder();
try {
rf = new RandomAccessFile(filename, "r");
long fileLength = rf.length();
long start = rf.getFilePointer();
long readIndex = start + fileLength - 1;
long minIndex = readIndex - maxSize;
if (minIndex < 0) {
minIndex = 0;
}
String line;
rf.seek(readIndex);
int k = 0;
int c = -1;
while (readIndex > start) {
@@ -48,7 +52,12 @@ public class FileUtils {
readIndex--;
}
readIndex--;
rf.seek(readIndex);
if (readIndex < minIndex) {
break;
} else {
rf.seek(readIndex);
}
if (readIndex == 0) {
readText = rf.readLine();
}
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java
+1
View File
@@ -148,6 +148,7 @@ public class ServerRunningMonitor extends AbstractCanalLifeCycle {
activeData = serverData;
processActiveEnter();// 触发一下事件
mutex.set(true);
release = false;
} catch (ZkNodeExistsException e) {
bytes = zkClient.readData(path, true);
if (bytes == null) {// 如果不存在节点，立即尝试一次
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java
+5 -1
View File
@@ -16,8 +16,12 @@ public class CanalConstants {
public static final String CANAL_IP = ROOT + "." + "ip";
public static final String CANAL_REGISTER_IP = ROOT + "." + "register.ip";
public static final String CANAL_PORT = ROOT + "." + "port";
public static final String CANAL_USER = ROOT + "." + "user";
public static final String CANAL_PASSWD = ROOT + "." + "passwd";
public static final String CANAL_METRICS_PULL_PORT = ROOT + "." + "metrics.pull.port";
public static final String CANAL_ADMIN_JMX_PORT = ROOT + "." + "admin.jmx.port";
public static final String CANAL_ADMIN_PORT = ROOT + "." + "admin.port";
public static final String CANAL_ADMIN_USER = ROOT + "." + "admin.user";
public static final String CANAL_ADMIN_PASSWD = ROOT + "." + "admin.passwd";
public static final String CANAL_ZKSERVERS = ROOT + "." + "zkServers";
public static final String CANAL_WITHOUT_NETTY = ROOT + "." + "withoutNetty";
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
+4
View File
@@ -121,6 +121,9 @@ public class CanalController {
embededCanalServer.setMetricsPort(11112);
}
embededCanalServer.setUser(getProperty(properties, CanalConstants.CANAL_USER));
embededCanalServer.setPasswd(getProperty(properties, CanalConstants.CANAL_PASSWD));
String canalWithoutNetty = getProperty(properties, CanalConstants.CANAL_WITHOUT_NETTY);
if (canalWithoutNetty == null || "false".equals(canalWithoutNetty)) {
canalServer = CanalServerWithNetty.instance();
@@ -571,4 +574,5 @@ public class CanalController {
public Map<String, InstanceConfig> getInstanceConfigs() {
return instanceConfigs;
}
}
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalLauncher.java
-23
View File
@@ -8,9 +8,6 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.deployer.mbean.CanalServerAgent;
import com.alibaba.otter.canal.deployer.mbean.CanalServerBean;
import com.alibaba.otter.canal.deployer.mbean.CanalServerMXBean;
import com.alibaba.otter.canal.deployer.monitor.remote.RemoteCanalConfigMonitor;
import com.alibaba.otter.canal.deployer.monitor.remote.RemoteConfigLoader;
import com.alibaba.otter.canal.deployer.monitor.remote.RemoteConfigLoaderFactory;
@@ -76,27 +73,7 @@ public class CanalLauncher {
});
}
CanalServerAgent canalServerAgent = null;
String jmxPort = properties.getProperty(CanalConstants.CANAL_ADMIN_JMX_PORT);
if (StringUtils.isNotEmpty(jmxPort)) {
String ip = properties.getProperty(CanalConstants.CANAL_IP);
String registerIp = properties.getProperty(CanalConstants.CANAL_REGISTER_IP);
if (StringUtils.isEmpty(registerIp)) {
// 兼容老的配置
registerIp = ip;
}
CanalServerMXBean canalServerMBean = new CanalServerBean(canalStater);
canalServerAgent = new CanalServerAgent(registerIp, Integer.parseInt(jmxPort), canalServerMBean);
Thread agentThread = new Thread(canalServerAgent::start);
agentThread.start();
}
runningLatch.await();
if (canalServerAgent != null) {
canalServerAgent.stop();
}
if (remoteConfigLoader != null) {
remoteConfigLoader.destroy();
}
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStater.java
+32 -1
View File
@@ -10,7 +10,9 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.admin.netty.CanalAdminWithNetty;
import com.alibaba.otter.canal.common.MQProperties;
import com.alibaba.otter.canal.deployer.admin.CanalAdminController;
import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
import com.alibaba.otter.canal.rocketmq.CanalRocketMQProducer;
import com.alibaba.otter.canal.server.CanalMQStarter;
@@ -36,6 +38,8 @@ public class CanalStater {
private volatile Properties properties;
private volatile boolean running = false;
private CanalAdminWithNetty canalAdmin;
public CanalStater(Properties properties){
this.properties = properties;
}
@@ -136,15 +140,42 @@ public class CanalStater {
controller.setCanalMQStarter(canalMQStarter);
}
// start canalAdmin
String port = properties.getProperty(CanalConstants.CANAL_ADMIN_PORT);
if (canalAdmin == null && StringUtils.isNotEmpty(port)) {
String user = properties.getProperty(CanalConstants.CANAL_ADMIN_USER);
String passwd = properties.getProperty(CanalConstants.CANAL_ADMIN_PASSWD);
CanalAdminController canalAdmin = new CanalAdminController(this);
canalAdmin.setUser(user);
canalAdmin.setPasswd(passwd);
String ip = properties.getProperty(CanalConstants.CANAL_IP);
CanalAdminWithNetty canalAdminWithNetty = CanalAdminWithNetty.instance();
canalAdminWithNetty.setCanalAdmin(canalAdmin);
canalAdminWithNetty.setPort(Integer.valueOf(port));
canalAdminWithNetty.setIp(ip);
canalAdminWithNetty.start();
this.canalAdmin = canalAdminWithNetty;
}
running = true;
}
public synchronized void stop() throws Throwable {
stop(false);
}
/**
* 销毁方法，远程配置变更时调用
*
* @throws Throwable
*/
public synchronized void stop() throws Throwable {
public synchronized void stop(boolean stopByAdmin) throws Throwable {
if (!stopByAdmin && canalAdmin != null) {
canalAdmin.stop();
canalAdmin = null;
}
if (controller != null) {
controller.stop();
controller = null;
deployer/src/main/java/com/alibaba/otter/canal/deployer/admin/CanalAdminController.java
+224
View File
@@ -0,0 +1,224 @@
package com.alibaba.otter.canal.deployer.admin;
import java.io.File;
import java.security.NoSuchAlgorithmException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.io.filefilter.TrueFileFilter;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.admin.CanalAdmin;
import com.alibaba.otter.canal.common.utils.FileUtils;
import com.alibaba.otter.canal.deployer.CanalController;
import com.alibaba.otter.canal.deployer.CanalStater;
import com.alibaba.otter.canal.deployer.InstanceConfig;
import com.alibaba.otter.canal.deployer.monitor.InstanceAction;
import com.alibaba.otter.canal.deployer.monitor.InstanceConfigMonitor;
import com.alibaba.otter.canal.deployer.monitor.SpringInstanceConfigMonitor;
import com.alibaba.otter.canal.instance.core.CanalInstance;
import com.alibaba.otter.canal.protocol.SecurityUtil;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.google.common.base.Joiner;
/**
* 提供canal admin的管理操作
*
* @author agapple 2019年8月24日 下午11:39:01
* @since 1.1.4
*/
public class CanalAdminController implements CanalAdmin {
private static final Logger logger = LoggerFactory.getLogger(CanalAdminController.class);
private String user;
private String passwd;
private CanalStater canalStater;
public CanalAdminController(CanalStater canalStater){
this.canalStater = canalStater;
}
@Override
public boolean check() {
return canalStater.isRunning();
}
@Override
public synchronized boolean start() {
try {
if (!canalStater.isRunning()) {
canalStater.start();
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}
@Override
public synchronized boolean stop() {
try {
if (canalStater.isRunning()) {
canalStater.stop(true);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}
@Override
public synchronized boolean restart() {
stop();
return start();
}
@Override
public boolean auth(String user, String passwd, byte[] seed) {
// 如果user/passwd密码为空,则任何用户账户都能登录
if ((StringUtils.isEmpty(this.user) || StringUtils.equals(this.user, user))) {
if (StringUtils.isEmpty(this.passwd)) {
return true;
} else if (StringUtils.isEmpty(passwd)) {
// 如果server密码有配置,客户端密码为空,则拒绝
return false;
}
try {
byte[] passForClient = SecurityUtil.hexStr2Bytes(passwd);
return SecurityUtil.scrambleServerAuth(passForClient, SecurityUtil.hexStr2Bytes(this.passwd), seed);
} catch (NoSuchAlgorithmException e) {
return false;
}
}
return false;
}
@Override
public String getRunningInstances() {
try {
CanalController controller = canalStater.getController();
if (controller != null) {
Map<String, InstanceConfig> instanceConfigs = controller.getInstanceConfigs();
if (instanceConfigs != null) {
return Joiner.on(",").join(instanceConfigs.keySet());
}
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return "";
}
@Override
public boolean checkInstance(String destination) {
Map<String, CanalInstance> instances = CanalServerWithEmbedded.instance().getCanalInstances();
if (instances == null || !instances.containsKey(destination)) {
return false;
} else {
CanalInstance instance = instances.get(destination);
return instance.isStart();
}
}
@Override
public boolean startInstance(String destination) {
try {
InstanceAction instanceAction = getInstanceAction(destination);
if (instanceAction != null) {
instanceAction.start(destination);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}
@Override
public boolean stopInstance(String destination) {
try {
InstanceAction instanceAction = getInstanceAction(destination);
if (instanceAction != null) {
instanceAction.stop(destination);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}
@Override
public boolean restartInstance(String destination) {
try {
InstanceAction instanceAction = getInstanceAction(destination);
if (instanceAction != null) {
instanceAction.reload(destination);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}
@Override
public String listCanalLog() {
Collection<File> files = org.apache.commons.io.FileUtils.listFiles(new File("../logs/canal/"),
TrueFileFilter.TRUE,
TrueFileFilter.TRUE);
List<String> names = files.stream().map(f -> f.getName()).collect(Collectors.toList());
return Joiner.on(",").join(names);
}
@Override
public String canalLog(int lines) {
return FileUtils.readFileFromOffset("../logs/canal/canal.log", lines, "UTF-8");
}
@Override
public String listInstanceLog(String destination) {
Collection<File> files = org.apache.commons.io.FileUtils.listFiles(new File("../logs/" + destination + "/"),
TrueFileFilter.TRUE,
TrueFileFilter.TRUE);
List<String> names = files.stream().map(f -> f.getName()).collect(Collectors.toList());
return Joiner.on(",").join(names);
}
@Override
public String instanceLog(String destination, String fileName, int lines) {
if (StringUtils.isEmpty(fileName)) {
fileName = destination + ".log";
}
return FileUtils.readFileFromOffset("../logs/" + destination + "/" + fileName, lines, "UTF-8");
}
private InstanceAction getInstanceAction(String destination) {
Map<InstanceConfig.InstanceMode, InstanceConfigMonitor> monitors = canalStater.getController()
.getInstanceConfigMonitors();
SpringInstanceConfigMonitor monitor = (SpringInstanceConfigMonitor) monitors.get(InstanceConfig.InstanceMode.SPRING);
Map<String, InstanceAction> instanceActions = monitor.getActions();
return instanceActions.get(destination);
}
public void setUser(String user) {
this.user = user;
}
public void setPasswd(String passwd) {
this.passwd = passwd;
}
public void setCanalStater(CanalStater canalStater) {
this.canalStater = canalStater;
}
}
deployer/src/main/java/com/alibaba/otter/canal/deployer/mbean/CanalServerAgent.java
-79
View File
@@ -1,79 +0,0 @@
package com.alibaba.otter.canal.deployer.mbean;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.rmi.registry.LocateRegistry;
import javax.management.MBeanServer;
import javax.management.Notification;
import javax.management.NotificationListener;
import javax.management.ObjectName;
import javax.management.remote.JMXConnectorServer;
import javax.management.remote.JMXConnectorServerFactory;
import javax.management.remote.JMXServiceURL;
/**
* Canal Server Agent 用于远程JMX调用
*
* @author rewerma 2019-07-13 下午05:12:16
* @version 1.0.0
*/
public class CanalServerAgent implements NotificationListener {
private static final Logger logger = LoggerFactory.getLogger(CanalServerBean.class);
private MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
private String ip = "0.0.0.0";
private int port;
private CanalServerMXBean canalServerMbean;
private volatile JMXConnectorServer cs;
public CanalServerAgent(String ip, int port, CanalServerMXBean canalServerMbean){
if (StringUtils.isNotEmpty(ip)) {
this.ip = ip;
}
this.port = port;
this.canalServerMbean = canalServerMbean;
}
@Override
public void handleNotification(Notification notification, Object handback) {
}
public synchronized void start() {
try {
if (cs == null) {
ObjectName name = new ObjectName("CanalServerAgent:type=CanalServerStatus");
mBeanServer.registerMBean(canalServerMbean, name);
LocateRegistry.createRegistry(port);
JMXServiceURL jmxServiceURL = new JMXServiceURL(
"service:jmx:rmi:///jndi/rmi://" + ip + ":" + port + "/jmxrmi");
cs = JMXConnectorServerFactory.newJMXConnectorServer(jmxServiceURL, null, mBeanServer);
cs.start();
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
public synchronized void stop() {
if (cs != null) {
try {
cs.stop();
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
}
}
}
deployer/src/main/java/com/alibaba/otter/canal/deployer/mbean/CanalServerBean.java
-156
View File
@@ -1,156 +0,0 @@
package com.alibaba.otter.canal.deployer.mbean;
import com.alibaba.otter.canal.common.utils.FileUtils;
import com.alibaba.otter.canal.deployer.CanalLauncher;
import com.alibaba.otter.canal.deployer.CanalStater;
import com.alibaba.otter.canal.deployer.InstanceConfig;
import com.alibaba.otter.canal.deployer.monitor.InstanceAction;
import com.alibaba.otter.canal.deployer.monitor.InstanceConfigMonitor;
import com.alibaba.otter.canal.deployer.monitor.SpringInstanceConfigMonitor;
import com.google.common.base.Joiner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.management.NotificationBroadcasterSupport;
import java.util.Map;
/**
* Canal配置信息业务层
*
* @author rewerma 2019-07-13 下午05:12:16
* @version 1.0.0
*/
public class CanalServerBean extends NotificationBroadcasterSupport implements CanalServerMXBean {
private static final Logger logger = LoggerFactory.getLogger(CanalServerBean.class);
private volatile int status;
private CanalStater canalStater;
public CanalServerBean(CanalStater canalStater){
this.canalStater = canalStater;
this.status = canalStater.isRunning() ? 1 : 0;
}
@Override
public int getStatus() {
return status;
}
@Override
public synchronized boolean start() {
try {
if (!canalStater.isRunning()) {
canalStater.start();
status = 1;
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}
@Override
public synchronized boolean stop() {
try {
if (canalStater.isRunning()) {
canalStater.stop();
status = 0;
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}
@Override
public synchronized boolean restart() {
stop();
return start();
}
@Override
public synchronized boolean exit() {
stop();
CanalLauncher.runningLatch.countDown();
return true;
}
@Override
public synchronized boolean startInstance(String destination) {
try {
InstanceAction instanceAction = getInstanceAction(destination);
if (instanceAction != null) {
instanceAction.start(destination);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}
@Override
public synchronized boolean stopInstance(String destination) {
try {
InstanceAction instanceAction = getInstanceAction(destination);
if (instanceAction != null) {
instanceAction.stop(destination);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}
@Override
public synchronized boolean reloadInstance(String destination) {
try {
InstanceAction instanceAction = getInstanceAction(destination);
if (instanceAction != null) {
instanceAction.reload(destination);
return true;
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return false;
}
@Override
public synchronized String getRunningInstances() {
try {
Map<String, InstanceConfig> instanceConfigs = canalStater.getController().getInstanceConfigs();
if (instanceConfigs != null) {
return Joiner.on(",").join(instanceConfigs.keySet());
}
} catch (Throwable e) {
logger.error(e.getMessage(), e);
}
return "";
}
@Override
public String canalLog() {
return FileUtils.readFileFromOffset("../logs/canal/canal.log", 100, "UTF-8");
}
@Override
public String instanceLog(String destination) {
return FileUtils.readFileFromOffset("../logs/" + destination + "/" + destination + ".log", 100, "UTF-8");
}
private InstanceAction getInstanceAction(String destination) {
Map<InstanceConfig.InstanceMode, InstanceConfigMonitor> monitors = canalStater.getController()
.getInstanceConfigMonitors();
SpringInstanceConfigMonitor monitor = (SpringInstanceConfigMonitor) monitors
.get(InstanceConfig.InstanceMode.SPRING);
Map<String, InstanceAction> instanceActions = monitor.getActions();
return instanceActions.get(destination);
}
}
deployer/src/main/java/com/alibaba/otter/canal/deployer/mbean/CanalServerMXBean.java
-90
View File
@@ -1,90 +0,0 @@
package com.alibaba.otter.canal.deployer.mbean;
/**
* Canal Server JMX MBean
*
* @author rewerma 2019-07-13 下午05:12:16
* @version 1.0.0
*/
public interface CanalServerMXBean {
/**
* 获取Canal Server状态
*
* @return 状态代码
*/
int getStatus();
/**
* 启动Canal Server
*
* @return 是否成功
*/
boolean start();
/**
* 停止Canal Server
*
* @return 是否成功
*/
boolean stop();
/**
* 重启Canal Server
*
* @return 是否成功
*/
boolean restart();
/**
* 退出Canal Server(关闭进程)
*
* @return 是否成功
*/
boolean exit();
/**
* 通过实例名启动实例
*
* @param destination 实例名
* @return 是否成功
*/
boolean startInstance(String destination);
/**
* 通过实例名关闭实例
*
* @param destination 实例名
* @return 是否成功
*/
boolean stopInstance(String destination);
/**
* 通过实例名重启实例
*
* @param destination 实例名
* @return 是否成功
*/
boolean reloadInstance(String destination);
/**
* 获取所有当前节点下运行中的实例
*
* @return 实例信息
*/
String getRunningInstances();
/**
* 获取Canal Server日志(末尾100行)
*
* @return 日志信息
*/
String canalLog();
/**
* 通过实例名获取实例日志(末尾100行)
*
* @return 日志信息
*/
String instanceLog(String destination);
}
deployer/src/main/resources/canal.properties
+10 -2
View File
@@ -11,7 +11,15 @@ canal.ip =
canal.register.ip =
canal.port = 11111
canal.metrics.pull.port = 11112
canal.admin.jmx.port = 11113
# canal instance user/passwd
# canal.user = canal
# canal.passwd = E3619321C1A937C46A0D8BD1DAC39F93B27D4458
# canal admin config
canal.admin.port = 11110
canal.admin.user = admin
canal.admin.passwd = 4ACFE3202A5FF5CF467898FC58AAB1D615029441
canal.zkServers =
# flush data to zk
canal.zookeeper.flush.period = 1000
@@ -131,4 +139,4 @@ canal.mq.accessChannel = local
##################################################
canal.mq.kafka.kerberos.enable = false
canal.mq.kafka.kerberos.krb5FilePath = "../conf/kerberos/krb5.conf"
canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf"
canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf"
driver/src/test/java/com/alibaba/otter/canal/parse/driver/mysql/MysqlConnectorTest.java
+1 -2
View File
@@ -4,11 +4,10 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import com.alibaba.otter.canal.parse.driver.mysql.packets.server.ResultSetPacket;
@Ignore
public class MysqlConnectorTest {
@Test
example/src/main/java/com/alibaba/otter/canal/example/AbstractCanalClientTest.java
+5
View File
@@ -79,6 +79,11 @@ public class AbstractCanalClientTest extends BaseCanalClientTest {
}
} catch (Exception e) {
logger.error("process error!", e);
try {
Thread.sleep(1000L);
} catch (InterruptedException e1) {
// ignore
}
} finally {
connector.disconnect();
MDC.remove("destination");
protocol/pom.xml
+6
View File
@@ -26,5 +26,11 @@
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
<!-- junit -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>
protocol/src/main/java/com/alibaba/otter/canal/protocol/AdminPacket.java
+6253
View File
File diff suppressed because it is too large Load Diff
protocol/src/main/java/com/alibaba/otter/canal/protocol/AdminProtocol.proto
+83
View File
@@ -0,0 +1,83 @@
syntax = "proto3";
package com.alibaba.otter.canal.protocol;
option java_package = "com.alibaba.otter.canal.protocol";
option java_outer_classname = "AdminPacket";
option optimize_for = SPEED;
enum PacketType {
//compatible
PACKAGETYPECOMPATIBLEPROTO2 = 0;
HANDSHAKE = 1;
CLIENTAUTHENTICATION = 2;
ACK = 3;
SERVER = 4;
INSTANCE = 5;
LOG = 6;
}
message Packet {
//[default = 17];
oneof magic_number_present {
int32 magic_number = 1;
}
// [default = 1]
oneof version_present {
int32 version = 2;
};
PacketType type = 3;
bytes body = 4;
}
message Ack {
// [default = 0]
oneof error_code_present {
int32 code = 1;
}
string message = 2; // if something like compression is not supported, erorr_message will tell about it.
}
message Handshake {
// [default = "utf8"];
oneof communication_encoding_present {
string communication_encoding = 1;
}
bytes seeds = 2;
}
// client authentication
message ClientAuth {
string username = 1;
bytes password = 2; // hashed password with seeds from Handshake message
// [default = 0]
oneof net_read_timeout_present {
int32 net_read_timeout = 3; // in seconds
}
// [default = 0];
oneof net_write_timeout_present {
int32 net_write_timeout = 4; // in seconds
}
}
message ServerAdmin {
string action = 1; // check/start/stop/restart/list
}
message InstanceAdmin {
string destination = 1;
string action = 2; // check/start/stop/reload
}
message LogAdmin {
string type = 1; // canal/instance
string action = 2;
oneof destination_present {
string destination = 3;
}
oneof file_present {
string file = 4;
}
oneof count_present {
int32 count = 5; // 默认tail 100行，最大不超过4MB
}
}
protocol/src/main/java/com/alibaba/otter/canal/protocol/CanalPacket.java
+5 -5
View File
@@ -630,7 +630,7 @@ public final class CanalPacket {
if (versionPresentCase_ == 2) {
return (java.lang.Integer) versionPresent_;
}
return 1;
return 0;
}
public static final int TYPE_FIELD_NUMBER = 3;
@@ -658,7 +658,7 @@ public final class CanalPacket {
if (compressionPresentCase_ == 4) {
return (java.lang.Integer) compressionPresent_;
}
return Compression.NONE_VALUE;
return 0;
}
/**
* <code>.com.alibaba.otter.canal.protocol.Compression compression = 4;</code>
@@ -1180,7 +1180,7 @@ public final class CanalPacket {
if (magicNumberPresentCase_ == 1) {
return (java.lang.Integer) magicNumberPresent_;
}
return 17;
return 0;
}
/**
* <code>int32 magic_number = 1;</code>
@@ -7731,7 +7731,7 @@ public final class CanalPacket {
if (timeoutPresentCase_ == 4) {
return (java.lang.Long) timeoutPresent_;
}
return -1L;
return 0L;
}
public static final int UNIT_FIELD_NUMBER = 5;
@@ -7746,7 +7746,7 @@ public final class CanalPacket {
if (unitPresentCase_ == 5) {
return (java.lang.Integer) unitPresent_;
}
return 2;
return 0;
}
public static final int AUTO_ACK_FIELD_NUMBER = 6;
protocol/src/main/java/com/alibaba/otter/canal/protocol/SecurityUtil.java
+223
View File
@@ -0,0 +1,223 @@
package com.alibaba.otter.canal.protocol;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
/**
* <pre>
* 1、client
* stage1_hash = SHA1(明文密码).
* token = SHA1(scramble + SHA1(stage1_hash)) XOR stage1_hash
* 2. server
* token = SHA1(token XOR SHA1(scramble + password))
* 3. checktoken vs password
* </pre>
*
* @author agapple 2019年8月26日 下午4:58:15
* @since 1.1.4
*/
public class SecurityUtil {
public static final String scrambleGenPass(byte[] pass) throws NoSuchAlgorithmException {
MessageDigest md = MessageDigest.getInstance("SHA-1");
byte[] pass1 = md.digest(pass);
md.reset();
byte[] pass2 = md.digest(pass1);
return SecurityUtil.byte2HexStr(pass2);
}
/**
* server auth check
*/
public static final boolean scrambleServerAuth(byte[] token, byte[] pass, byte[] seed)
throws NoSuchAlgorithmException {
MessageDigest md = MessageDigest.getInstance("SHA-1");
md.update(seed);
byte[] pass1 = md.digest(pass);
for (int i = 0; i < pass1.length; i++) {
pass1[i] = (byte) (token[i] ^ pass1[i]);
}
md = MessageDigest.getInstance("SHA-1");
byte[] pass2 = md.digest(pass1);
return Arrays.equals(pass, pass2);
}
public static final byte[] scramble411(byte[] pass, byte[] seed) throws NoSuchAlgorithmException {
MessageDigest md = MessageDigest.getInstance("SHA-1");
byte[] pass1 = md.digest(pass);
md.reset();
byte[] pass2 = md.digest(pass1);
md.reset();
md.update(seed);
byte[] pass3 = md.digest(pass2);
for (int i = 0; i < pass3.length; i++) {
pass3[i] = (byte) (pass3[i] ^ pass1[i]);
}
return pass3;
}
/**
* bytes转换成十六进制字符串
*/
public static String byte2HexStr(byte[] b) {
StringBuilder hs = new StringBuilder();
for (int n = 0; n < b.length; n++) {
String hex = (Integer.toHexString(b[n] & 0XFF));
if (hex.length() == 1) {
hs.append("0" + hex);
} else {
hs.append(hex);
}
}
return hs.toString();
}
/**
* bytes转换成十六进制字符串
*/
public static byte[] hexStr2Bytes(String src) {
if (src == null) {
return null;
}
int offset = 0;
int length = src.length();
if (length == 0) {
return new byte[0];
}
boolean odd = length << 31 == Integer.MIN_VALUE;
byte[] bs = new byte[odd ? (length + 1) >> 1 : length >> 1];
for (int i = offset, limit = offset + length; i < limit; ++i) {
char high, low;
if (i == offset && odd) {
high = '0';
low = src.charAt(i);
} else {
high = src.charAt(i);
low = src.charAt(++i);
}
int b;
switch (high) {
case '0':
b = 0;
break;
case '1':
b = 0x10;
break;
case '2':
b = 0x20;
break;
case '3':
b = 0x30;
break;
case '4':
b = 0x40;
break;
case '5':
b = 0x50;
break;
case '6':
b = 0x60;
break;
case '7':
b = 0x70;
break;
case '8':
b = 0x80;
break;
case '9':
b = 0x90;
break;
case 'a':
case 'A':
b = 0xa0;
break;
case 'b':
case 'B':
b = 0xb0;
break;
case 'c':
case 'C':
b = 0xc0;
break;
case 'd':
case 'D':
b = 0xd0;
break;
case 'e':
case 'E':
b = 0xe0;
break;
case 'f':
case 'F':
b = 0xf0;
break;
default:
throw new IllegalArgumentException("illegal hex-string: " + src);
}
switch (low) {
case '0':
break;
case '1':
b += 1;
break;
case '2':
b += 2;
break;
case '3':
b += 3;
break;
case '4':
b += 4;
break;
case '5':
b += 5;
break;
case '6':
b += 6;
break;
case '7':
b += 7;
break;
case '8':
b += 8;
break;
case '9':
b += 9;
break;
case 'a':
case 'A':
b += 10;
break;
case 'b':
case 'B':
b += 11;
break;
case 'c':
case 'C':
b += 12;
break;
case 'd':
case 'D':
b += 13;
break;
case 'e':
case 'E':
b += 14;
break;
case 'f':
case 'F':
b += 15;
break;
default:
throw new IllegalArgumentException("illegal hex-string: " + src);
}
bs[(i - offset) >> 1] = (byte) b;
}
return bs;
}
}
canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/jmx/CanalServerMXBean.java → server/src/main/java/com/alibaba/otter/canal/admin/CanalAdmin.java
+43 -23
View File
@@ -1,19 +1,24 @@
package com.alibaba.otter.canal.admin.jmx;
package com.alibaba.otter.canal.admin;
/**
* Canal Server JMX MBean
*
* @author rewerma 2019-07-13 下午05:12:16
* @version 1.0.0
* Canal Admin动态管理接口
*
* @author agapple 2019年8月24日 下午9:45:49
* @since 1.1.4
*/
public interface CanalServerMXBean {
public interface CanalAdmin {
/**
* 获取Canal Server状态
* 校验账号密码
*/
boolean auth(String user, String passwd, byte[] seed);
/**
* 获取Canal Server状态, 1代表运行/0代表不运行
*
* @return 状态代码
*/
int getStatus();
boolean check();
/**
* 启动Canal Server
@@ -37,11 +42,19 @@ public interface CanalServerMXBean {
boolean restart();
/**
* 退出Canal Server(关闭进程)
* 获取所有当前节点下运行中的实例
*
* @return 是否成功
* @return 实例信息
*/
boolean exit();
String getRunningInstances();
/**
* 通过实例名检查
*
* @param destination
* @return
*/
boolean checkInstance(String destination);
/**
* 通过实例名启动实例
@@ -65,26 +78,33 @@ public interface CanalServerMXBean {
* @param destination 实例名
* @return 是否成功
*/
boolean reloadInstance(String destination);
boolean restartInstance(String destination);
/**
* 获取所有当前节点下运行中的实例
*
* @return 实例信息
*/
String getRunningInstances();
/**
* 获取Canal Server日志(末尾100行)
* 获取Canal Server日志列表
*
* @return 日志信息
*/
String canalLog();
String listCanalLog();
/**
* 通过实例名获取实例日志(末尾100行)
* 获取Canal Server日志
*
* @return 日志信息
*/
String instanceLog(String destination);
String canalLog(int lines);
/**
* 获取Instance的机器日志列表
*
* @param destination
*/
String listInstanceLog(String destination);
/**
* 通过实例名获取实例日志
*
* @return 日志信息
*/
String instanceLog(String destination, String fileName, int lines);
}
server/src/main/java/com/alibaba/otter/canal/admin/handler/ClientAuthenticationHandler.java
+119
View File
@@ -0,0 +1,119 @@
package com.alibaba.otter.canal.admin.handler;
import java.util.concurrent.TimeUnit;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
import org.jboss.netty.handler.timeout.IdleStateAwareChannelHandler;
import org.jboss.netty.handler.timeout.IdleStateEvent;
import org.jboss.netty.handler.timeout.IdleStateHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.helpers.MessageFormatter;
import com.alibaba.otter.canal.admin.CanalAdmin;
import com.alibaba.otter.canal.admin.netty.AdminNettyUtils;
import com.alibaba.otter.canal.protocol.AdminPacket.ClientAuth;
import com.alibaba.otter.canal.protocol.AdminPacket.Packet;
import com.alibaba.otter.canal.server.netty.NettyUtils;
/**
* 客户端身份认证处理
*
* @author agapple 2019年8月24日 下午10:58:53
* @since 1.1.4
*/
public class ClientAuthenticationHandler extends SimpleChannelHandler {
private static final Logger logger = LoggerFactory.getLogger(ClientAuthenticationHandler.class);
private final int SUPPORTED_VERSION = 3;
private final int defaultSubscriptorDisconnectIdleTimeout = 60 * 60 * 1000;
private CanalAdmin canalAdmin;
private byte[] seed;
public ClientAuthenticationHandler(){
}
public ClientAuthenticationHandler(CanalAdmin canalAdmin){
this.canalAdmin = canalAdmin;
}
public void messageReceived(final ChannelHandlerContext ctx, MessageEvent e) throws Exception {
ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
final Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
switch (packet.getVersion()) {
case SUPPORTED_VERSION:
default:
final ClientAuth clientAuth = ClientAuth.parseFrom(packet.getBody());
if (seed == null) {
byte[] errorBytes = AdminNettyUtils.errorPacket(300,
MessageFormatter.format("auth failed for seed is null", clientAuth.getUsername()).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
}
if (!canalAdmin.auth(clientAuth.getUsername(), clientAuth.getPassword().toStringUtf8(), seed)) {
byte[] errorBytes = AdminNettyUtils.errorPacket(300,
MessageFormatter.format("auth failed for user:{}", clientAuth.getUsername()).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
}
byte[] ackBytes = AdminNettyUtils.ackPacket();
AdminNettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureListener() {
public void operationComplete(ChannelFuture future) throws Exception {
logger.info("remove unused channel handlers after authentication is done successfully.");
ctx.getPipeline().remove(HandshakeInitializationHandler.class.getName());
ctx.getPipeline().remove(ClientAuthenticationHandler.class.getName());
int readTimeout = defaultSubscriptorDisconnectIdleTimeout;
int writeTimeout = defaultSubscriptorDisconnectIdleTimeout;
if (clientAuth.getNetReadTimeout() > 0) {
readTimeout = clientAuth.getNetReadTimeout();
}
if (clientAuth.getNetWriteTimeout() > 0) {
writeTimeout = clientAuth.getNetWriteTimeout();
}
// fix bug: soTimeout parameter's unit from connector is
// millseconds.
IdleStateHandler idleStateHandler = new IdleStateHandler(NettyUtils.hashedWheelTimer,
readTimeout,
writeTimeout,
0,
TimeUnit.MILLISECONDS);
ctx.getPipeline().addBefore(SessionHandler.class.getName(),
IdleStateHandler.class.getName(),
idleStateHandler);
IdleStateAwareChannelHandler idleStateAwareChannelHandler = new IdleStateAwareChannelHandler() {
public void channelIdle(ChannelHandlerContext ctx, IdleStateEvent e) throws Exception {
logger.warn("channel:{} idle timeout exceeds, close channel to save server resources...",
ctx.getChannel());
ctx.getChannel().close();
}
};
ctx.getPipeline().addBefore(SessionHandler.class.getName(),
IdleStateAwareChannelHandler.class.getName(),
idleStateAwareChannelHandler);
}
});
break;
}
}
public void setCanalAdmin(CanalAdmin canalAdmin) {
this.canalAdmin = canalAdmin;
}
public void setSeed(byte[] seed) {
this.seed = seed;
}
}
server/src/main/java/com/alibaba/otter/canal/admin/handler/HandshakeInitializationHandler.java
+62
View File
@@ -0,0 +1,62 @@
package com.alibaba.otter.canal.admin.handler;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
import org.jboss.netty.channel.group.ChannelGroup;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.otter.canal.admin.netty.AdminNettyUtils;
import com.alibaba.otter.canal.protocol.AdminPacket;
import com.alibaba.otter.canal.protocol.AdminPacket.Handshake;
import com.alibaba.otter.canal.protocol.AdminPacket.Packet;
import com.google.protobuf.ByteString;
/**
* handshake交互
*
* @author agapple 2019年8月24日 下午10:58:34
* @since 1.1.4
*/
public class HandshakeInitializationHandler extends SimpleChannelHandler {
// support to maintain socket channel.
private ChannelGroup childGroups;
public HandshakeInitializationHandler(ChannelGroup childGroups){
this.childGroups = childGroups;
}
private static final Logger logger = LoggerFactory.getLogger(HandshakeInitializationHandler.class);
public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
// add new socket channel in channel container, used to manage sockets.
if (childGroups != null) {
childGroups.add(ctx.getChannel());
}
final byte[] seed = org.apache.commons.lang3.RandomUtils.nextBytes(8);
byte[] body = Packet.newBuilder()
.setType(AdminPacket.PacketType.HANDSHAKE)
.setVersion(AdminNettyUtils.VERSION)
.setBody(Handshake.newBuilder().setSeeds(ByteString.copyFrom(seed)).build().toByteString())
.build()
.toByteArray();
AdminNettyUtils.write(ctx.getChannel(), body, new ChannelFutureListener() {
public void operationComplete(ChannelFuture future) throws Exception {
logger.info("remove unused channel handlers after authentication is done successfully.");
ctx.getPipeline().get(HandshakeInitializationHandler.class.getName());
ClientAuthenticationHandler handler = (ClientAuthenticationHandler) ctx.getPipeline()
.get(ClientAuthenticationHandler.class.getName());
handler.setSeed(seed);
}
});
logger.info("send handshake initialization packet to : {}", ctx.getChannel());
}
}
server/src/main/java/com/alibaba/otter/canal/admin/handler/SessionHandler.java
+154
View File
@@ -0,0 +1,154 @@
package com.alibaba.otter.canal.admin.handler;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.ExceptionEvent;
import org.jboss.netty.channel.MessageEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.helpers.MessageFormatter;
import com.alibaba.otter.canal.admin.CanalAdmin;
import com.alibaba.otter.canal.admin.netty.AdminNettyUtils;
import com.alibaba.otter.canal.protocol.AdminPacket.InstanceAdmin;
import com.alibaba.otter.canal.protocol.AdminPacket.LogAdmin;
import com.alibaba.otter.canal.protocol.AdminPacket.Packet;
import com.alibaba.otter.canal.protocol.AdminPacket.ServerAdmin;
public class SessionHandler extends SimpleChannelHandler {
private static final Logger logger = LoggerFactory.getLogger(SessionHandler.class);
private CanalAdmin canalAdmin;
public SessionHandler(){
}
public SessionHandler(CanalAdmin canalAdmin){
this.canalAdmin = canalAdmin;
}
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
logger.info("message receives in session handler...");
ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
try {
String action = null;
String message = null;
String destination = null;
switch (packet.getType()) {
case SERVER:
ServerAdmin serverAdmin = ServerAdmin.parseFrom(packet.getBody());
action = serverAdmin.getAction();
switch (action) {
case "check":
message = canalAdmin.check() ? "1" : "0";
break;
case "start":
message = canalAdmin.start() ? "1" : "0";
break;
case "stop":
message = canalAdmin.stop() ? "1" : "0";
break;
case "restart":
message = canalAdmin.restart() ? "1" : "0";
break;
case "list":
message = canalAdmin.getRunningInstances();
break;
default:
byte[] errorBytes = AdminNettyUtils.errorPacket(301,
MessageFormatter.format("ServerAdmin action={} is unknown", action).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
break;
}
AdminNettyUtils.write(ctx.getChannel(), AdminNettyUtils.ackPacket(message));
break;
case INSTANCE:
InstanceAdmin instanceAdmin = InstanceAdmin.parseFrom(packet.getBody());
destination = instanceAdmin.getDestination();
action = instanceAdmin.getAction();
switch (action) {
case "check":
message = canalAdmin.checkInstance(destination) ? "1" : "0";
break;
case "start":
message = canalAdmin.startInstance(destination) ? "1" : "0";
break;
case "stop":
message = canalAdmin.stopInstance(destination) ? "1" : "0";
break;
case "restart":
message = canalAdmin.restartInstance(destination) ? "1" : "0";
break;
default:
byte[] errorBytes = AdminNettyUtils.errorPacket(301,
MessageFormatter.format("InstanceAdmin action={} is unknown", action).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
break;
}
AdminNettyUtils.write(ctx.getChannel(), AdminNettyUtils.ackPacket(message));
break;
case LOG:
LogAdmin logAdmin = LogAdmin.parseFrom(packet.getBody());
action = logAdmin.getAction();
destination = logAdmin.getDestination();
String type = logAdmin.getType();
String file = logAdmin.getFile();
int count = logAdmin.getCount();
switch (type) {
case "server":
if ("list".equalsIgnoreCase(action)) {
message = canalAdmin.listCanalLog();
} else {
message = canalAdmin.canalLog(count);
}
break;
case "instance":
if ("list".equalsIgnoreCase(action)) {
message = canalAdmin.listInstanceLog(destination);
} else {
message = canalAdmin.instanceLog(destination, file, count);
}
break;
default:
byte[] errorBytes = AdminNettyUtils.errorPacket(301,
MessageFormatter.format("LogAdmin type={} is unknown", type).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
break;
}
AdminNettyUtils.write(ctx.getChannel(), AdminNettyUtils.ackPacket(message));
break;
default:
byte[] errorBytes = AdminNettyUtils.errorPacket(300,
MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
break;
}
} catch (Throwable exception) {
byte[] errorBytes = AdminNettyUtils.errorPacket(400,
MessageFormatter.format("something goes wrong with channel:{}, exception={}",
ctx.getChannel(),
ExceptionUtils.getStackTrace(exception)).getMessage());
AdminNettyUtils.write(ctx.getChannel(), errorBytes);
}
}
public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
logger.error("something goes wrong with channel:{}, exception={}",
ctx.getChannel(),
ExceptionUtils.getStackTrace(e.getCause()));
ctx.getChannel().close();
}
public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
}
public void setCanalAdmin(CanalAdmin canalAdmin) {
this.canalAdmin = canalAdmin;
}
}
server/src/main/java/com/alibaba/otter/canal/admin/netty/AdminNettyUtils.java
+63
View File
@@ -0,0 +1,63 @@
package com.alibaba.otter.canal.admin.netty;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.List;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.buffer.ChannelBuffers;
import org.jboss.netty.buffer.CompositeChannelBuffer;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.Channels;
import com.alibaba.otter.canal.protocol.AdminPacket;
import com.alibaba.otter.canal.protocol.AdminPacket.Ack;
import com.alibaba.otter.canal.protocol.AdminPacket.Packet;
public class AdminNettyUtils {
public static int HEADER_LENGTH = 4;
public static int VERSION = 1;
public static void write(Channel channel, ByteBuffer body) {
byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.limit()).array();
List<ChannelBuffer> components = new ArrayList<ChannelBuffer>(2);
components.add(ChannelBuffers.wrappedBuffer(ByteOrder.BIG_ENDIAN, header));
components.add(ChannelBuffers.wrappedBuffer(body));
Channels.write(channel, new CompositeChannelBuffer(ByteOrder.BIG_ENDIAN, components));
}
public static void write(Channel channel, byte[] body) {
byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.length).array();
Channels.write(channel, ChannelBuffers.wrappedBuffer(header, body));
}
public static void write(Channel channel, byte[] body, ChannelFutureListener channelFutureListner) {
byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.length).array();
Channels.write(channel, ChannelBuffers.wrappedBuffer(header, body)).addListener(channelFutureListner);
}
public static byte[] ackPacket() {
return ackPacket(null);
}
public static byte[] ackPacket(String message) {
return Packet.newBuilder()
.setType(AdminPacket.PacketType.ACK)
.setVersion(VERSION)
.setBody(Ack.newBuilder().setCode(0).setMessage(message == null ? "" : message).build().toByteString())
.build()
.toByteArray();
}
public static byte[] errorPacket(int errorCode, String errorMessage) {
return Packet.newBuilder()
.setType(AdminPacket.PacketType.ACK)
.setVersion(VERSION)
.setBody(Ack.newBuilder().setCode(errorCode).setMessage(errorMessage).build().toByteString())
.build()
.toByteArray();
}
}
server/src/main/java/com/alibaba/otter/canal/admin/netty/CanalAdminWithNetty.java
+127
View File
@@ -0,0 +1,127 @@
package com.alibaba.otter.canal.admin.netty;
import java.net.InetSocketAddress;
import java.util.concurrent.Executors;
import org.apache.commons.lang.StringUtils;
import org.jboss.netty.bootstrap.ServerBootstrap;
import org.jboss.netty.channel.Channel;
import org.jboss.netty.channel.ChannelPipeline;
import org.jboss.netty.channel.ChannelPipelineFactory;
import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.group.ChannelGroup;
import org.jboss.netty.channel.group.DefaultChannelGroup;
import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
import com.alibaba.otter.canal.admin.CanalAdmin;
import com.alibaba.otter.canal.admin.handler.ClientAuthenticationHandler;
import com.alibaba.otter.canal.admin.handler.HandshakeInitializationHandler;
import com.alibaba.otter.canal.admin.handler.SessionHandler;
import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
import com.alibaba.otter.canal.server.netty.handler.FixedHeaderFrameDecoder;
/**
* 基于netty网络服务的server实现
*
* @author jianghang 2012-7-12 下午01:34:49
* @version 1.0.0
*/
public class CanalAdminWithNetty extends AbstractCanalLifeCycle {
private String ip;
private int port;
private Channel serverChannel = null;
private ServerBootstrap bootstrap = null;
private ChannelGroup childGroups = null; // socket channel
// container, used to
// close sockets
// explicitly.
private CanalAdmin canalAdmin;
private static class SingletonHolder {
private static final CanalAdminWithNetty CANAL_ADMIN_WITH_NETTY = new CanalAdminWithNetty();
}
private CanalAdminWithNetty(){
this.childGroups = new DefaultChannelGroup();
}
public static CanalAdminWithNetty instance() {
return SingletonHolder.CANAL_ADMIN_WITH_NETTY;
}
public void start() {
super.start();
this.bootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(Executors.newCachedThreadPool(),
Executors.newCachedThreadPool()));
/*
* enable keep-alive mechanism, handle abnormal network connection
* scenarios on OS level. the threshold parameters are depended on OS.
* e.g. On Linux: net.ipv4.tcp_keepalive_time = 300
* net.ipv4.tcp_keepalive_probes = 2 net.ipv4.tcp_keepalive_intvl = 30
*/
bootstrap.setOption("child.keepAlive", true);
/*
* optional parameter.
*/
bootstrap.setOption("child.tcpNoDelay", true);
// 构造对应的pipeline
bootstrap.setPipelineFactory(new ChannelPipelineFactory() {
public ChannelPipeline getPipeline() throws Exception {
ChannelPipeline pipelines = Channels.pipeline();
pipelines.addLast(FixedHeaderFrameDecoder.class.getName(), new FixedHeaderFrameDecoder());
// support to maintain child socket channel.
pipelines.addLast(HandshakeInitializationHandler.class.getName(),
new HandshakeInitializationHandler(childGroups));
pipelines.addLast(ClientAuthenticationHandler.class.getName(),
new ClientAuthenticationHandler(canalAdmin));
SessionHandler sessionHandler = new SessionHandler(canalAdmin);
pipelines.addLast(SessionHandler.class.getName(), sessionHandler);
return pipelines;
}
});
// 启动
if (StringUtils.isNotEmpty(ip)) {
this.serverChannel = bootstrap.bind(new InetSocketAddress(this.ip, this.port));
} else {
this.serverChannel = bootstrap.bind(new InetSocketAddress(this.port));
}
}
public void stop() {
super.stop();
if (this.serverChannel != null) {
this.serverChannel.close().awaitUninterruptibly(1000);
}
// close sockets explicitly to reduce socket channel hung in complicated
// network environment.
if (this.childGroups != null) {
this.childGroups.close().awaitUninterruptibly(5000);
}
if (this.bootstrap != null) {
this.bootstrap.releaseExternalResources();
}
}
public void setIp(String ip) {
this.ip = ip;
}
public void setPort(int port) {
this.port = port;
}
public void setCanalAdmin(CanalAdmin canalAdmin) {
this.canalAdmin = canalAdmin;
}
}
server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java
+2
View File
@@ -9,6 +9,7 @@ import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.server.netty.NettyUtils;
import com.google.protobuf.ByteString;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.WireFormat;
@@ -66,6 +67,7 @@ public class CanalMessageSerializer {
CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
packetBuilder.setType(PacketType.MESSAGES);
packetBuilder.setVersion(NettyUtils.VERSION);
packetBuilder.setBody(messageBuilder.build().toByteString());
return packetBuilder.build().toByteArray();
}
server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java
+35
View File
@@ -1,12 +1,15 @@
package com.alibaba.otter.canal.server.embedded;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
@@ -18,6 +21,7 @@ import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
import com.alibaba.otter.canal.protocol.CanalEntry;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.Message;
import com.alibaba.otter.canal.protocol.SecurityUtil;
import com.alibaba.otter.canal.protocol.position.LogPosition;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
@@ -52,6 +56,8 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
private CanalInstanceGenerator canalInstanceGenerator;
private int metricsPort;
private CanalMetricsService metrics = NopCanalMetricsService.NOP;
private String user;
private String passwd;
private static class SingletonHolder {
@@ -106,6 +112,27 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
metrics.terminate();
}
public boolean auth(String user, String passwd, byte[] seed) {
// 如果user/passwd密码为空,则任何用户账户都能登录
if ((StringUtils.isEmpty(this.user) || StringUtils.equals(this.user, user))) {
if (StringUtils.isEmpty(this.passwd)) {
return true;
} else if (StringUtils.isEmpty(passwd)) {
// 如果server密码有配置,客户端密码为空,则拒绝
return false;
}
try {
byte[] passForClient = SecurityUtil.hexStr2Bytes(passwd);
return SecurityUtil.scrambleServerAuth(passForClient, SecurityUtil.hexStr2Bytes(this.passwd), seed);
} catch (NoSuchAlgorithmException e) {
return false;
}
}
return false;
}
public void start(final String destination) {
final CanalInstance canalInstance = canalInstances.get(destination);
if (!canalInstance.isStart()) {
@@ -572,4 +599,12 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
this.metricsPort = metricsPort;
}
public void setUser(String user) {
this.user = user;
}
public void setPasswd(String passwd) {
this.passwd = passwd;
}
}
server/src/main/java/com/alibaba/otter/canal/server/netty/NettyUtils.java
+13 -8
View File
@@ -25,6 +25,7 @@ public class NettyUtils {
private static final Logger logger = LoggerFactory.getLogger(NettyUtils.class);
public static int HEADER_LENGTH = 4;
public static Timer hashedWheelTimer = new HashedWheelTimer();
public static int VERSION = 1;
public static void write(Channel channel, ByteBuffer body, ChannelFutureListener channelFutureListner) {
byte[] header = ByteBuffer.allocate(HEADER_LENGTH).order(ByteOrder.BIG_ENDIAN).putInt(body.limit()).array();
@@ -53,6 +54,7 @@ public class NettyUtils {
write(channel,
Packet.newBuilder()
.setType(CanalPacket.PacketType.ACK)
.setVersion(VERSION)
.setBody(Ack.newBuilder().build().toByteString())
.build()
.toByteArray(),
@@ -69,6 +71,7 @@ public class NettyUtils {
write(channel,
Packet.newBuilder()
.setType(CanalPacket.PacketType.ACK)
.setVersion(VERSION)
.setBody(Ack.newBuilder().setErrorCode(errorCode).setErrorMessage(errorMessage).build().toByteString())
.build()
.toByteArray(),
@@ -77,17 +80,19 @@ public class NettyUtils {
public static byte[] ackPacket() {
return Packet.newBuilder()
.setType(CanalPacket.PacketType.ACK)
.setBody(Ack.newBuilder().build().toByteString())
.build()
.toByteArray();
.setType(CanalPacket.PacketType.ACK)
.setVersion(VERSION)
.setBody(Ack.newBuilder().build().toByteString())
.build()
.toByteArray();
}
public static byte[] errorPacket(int errorCode, String errorMessage) {
return Packet.newBuilder()
.setType(CanalPacket.PacketType.ACK)
.setBody(Ack.newBuilder().setErrorCode(errorCode).setErrorMessage(errorMessage).build().toByteString())
.build()
.toByteArray();
.setType(CanalPacket.PacketType.ACK)
.setVersion(VERSION)
.setBody(Ack.newBuilder().setErrorCode(errorCode).setErrorMessage(errorMessage).build().toByteString())
.build()
.toByteArray();
}
}
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/ClientAuthenticationHandler.java
+18 -1
View File
@@ -15,6 +15,7 @@ import org.jboss.netty.handler.timeout.IdleStateHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.slf4j.helpers.MessageFormatter;
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor;
import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors;
@@ -36,6 +37,7 @@ public class ClientAuthenticationHandler extends SimpleChannelHandler {
private final int SUPPORTED_VERSION = 3;
private final int defaultSubscriptorDisconnectIdleTimeout = 60 * 60 * 1000;
private CanalServerWithEmbedded embeddedServer;
private byte[] seed;
public ClientAuthenticationHandler(){
@@ -52,6 +54,18 @@ public class ClientAuthenticationHandler extends SimpleChannelHandler {
case SUPPORTED_VERSION:
default:
final ClientAuth clientAuth = ClientAuth.parseFrom(packet.getBody());
if (seed == null) {
byte[] errorBytes = NettyUtils.errorPacket(400,
MessageFormatter.format("auth failed for seed is null", clientAuth.getUsername()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, null);
}
if (!embeddedServer.auth(clientAuth.getUsername(), clientAuth.getPassword().toStringUtf8(), seed)) {
byte[] errorBytes = NettyUtils.errorPacket(400,
MessageFormatter.format("auth failed for user:{}", clientAuth.getUsername()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, null);
}
// 如果存在订阅信息
if (StringUtils.isNotEmpty(clientAuth.getDestination())
&& StringUtils.isNotEmpty(clientAuth.getClientId())) {
@@ -61,7 +75,6 @@ public class ClientAuthenticationHandler extends SimpleChannelHandler {
try {
MDC.put("destination", clientIdentity.getDestination());
embeddedServer.subscribe(clientIdentity);
ctx.setAttachment(clientIdentity);// 设置状态数据
// 尝试启动，如果已经启动，忽略
if (!embeddedServer.isStart(clientIdentity.getDestination())) {
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
@@ -123,4 +136,8 @@ public class ClientAuthenticationHandler extends SimpleChannelHandler {
this.embeddedServer = embeddedServer;
}
public void setSeed(byte[] seed) {
this.seed = seed;
}
}
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/HandshakeInitializationHandler.java
+17 -2
View File
@@ -1,5 +1,7 @@
package com.alibaba.otter.canal.server.netty.handler;
import org.jboss.netty.channel.ChannelFuture;
import org.jboss.netty.channel.ChannelFutureListener;
import org.jboss.netty.channel.ChannelHandlerContext;
import org.jboss.netty.channel.ChannelStateEvent;
import org.jboss.netty.channel.SimpleChannelHandler;
@@ -11,6 +13,7 @@ import com.alibaba.otter.canal.protocol.CanalPacket;
import com.alibaba.otter.canal.protocol.CanalPacket.Handshake;
import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
import com.alibaba.otter.canal.server.netty.NettyUtils;
import com.google.protobuf.ByteString;
/**
* handshake交互
@@ -35,12 +38,24 @@ public class HandshakeInitializationHandler extends SimpleChannelHandler {
childGroups.add(ctx.getChannel());
}
final byte[] seed = org.apache.commons.lang3.RandomUtils.nextBytes(8);
byte[] body = Packet.newBuilder()
.setType(CanalPacket.PacketType.HANDSHAKE)
.setBody(Handshake.newBuilder().build().toByteString())
.setVersion(NettyUtils.VERSION)
.setBody(Handshake.newBuilder().setSeeds(ByteString.copyFrom(seed)).build().toByteString())
.build()
.toByteArray();
NettyUtils.write(ctx.getChannel(), body, null);
NettyUtils.write(ctx.getChannel(), body, new ChannelFutureListener() {
public void operationComplete(ChannelFuture future) throws Exception {
ctx.getPipeline().get(HandshakeInitializationHandler.class.getName());
ClientAuthenticationHandler handler = (ClientAuthenticationHandler) ctx.getPipeline()
.get(ClientAuthenticationHandler.class.getName());
handler.setSeed(seed);
}
});
logger.info("send handshake initialization packet to : {}", ctx.getChannel());
}
}
server/src/main/java/com/alibaba/otter/canal/server/netty/handler/SessionHandler.java
+107 -34
View File
@@ -79,11 +79,21 @@ public class SessionHandler extends SimpleChannelHandler {
// ctx.setAttachment(clientIdentity);// 设置状态数据
byte[] ackBytes = NettyUtils.ackPacket();
NettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureAggregator(sub.getDestination(),
sub, packet.getType(), ackBytes.length, System.nanoTime() - start));
sub,
packet.getType(),
ackBytes.length,
System.nanoTime() - start));
} else {
byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes ,new ChannelFutureAggregator(sub.getDestination(),
sub, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(sub.getDestination(),
sub,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
case UNSUBSCRIPTION:
@@ -96,12 +106,24 @@ public class SessionHandler extends SimpleChannelHandler {
embeddedServer.unsubscribe(clientIdentity);
stopCanalInstanceIfNecessary(clientIdentity);// 尝试关闭
byte[] ackBytes = NettyUtils.ackPacket();
NettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureAggregator(unsub.getDestination(),
unsub, packet.getType(), ackBytes.length, System.nanoTime() - start));
NettyUtils.write(ctx.getChannel(),
ackBytes,
new ChannelFutureAggregator(unsub.getDestination(),
unsub,
packet.getType(),
ackBytes.length,
System.nanoTime() - start));
} else {
byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(unsub.getDestination(),
unsub, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(unsub.getDestination(),
unsub,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
case GET:
@@ -177,15 +199,19 @@ public class SessionHandler extends SimpleChannelHandler {
}
output.checkNoSpaceLeft();
NettyUtils.write(ctx.getChannel(), body, new ChannelFutureAggregator(get.getDestination(),
get, packet.getType(), body.length, System.nanoTime() - start, message.getId() == -1));
get,
packet.getType(),
body.length,
System.nanoTime() - start,
message.getId() == -1));
// output.flush();
// byteBuffer.flip();
// NettyUtils.write(ctx.getChannel(), byteBuffer,
// null);
} else {
Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
packetBuilder.setType(PacketType.MESSAGES);
packetBuilder.setType(PacketType.MESSAGES).setVersion(NettyUtils.VERSION);
Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
messageBuilder.setBatchId(message.getId());
@@ -198,14 +224,27 @@ public class SessionHandler extends SimpleChannelHandler {
}
}
}
byte[] body = packetBuilder.setBody(messageBuilder.build().toByteString()).build().toByteArray();
byte[] body = packetBuilder.setBody(messageBuilder.build().toByteString())
.build()
.toByteArray();
NettyUtils.write(ctx.getChannel(), body, new ChannelFutureAggregator(get.getDestination(),
get, packet.getType(), body.length, System.nanoTime() - start, message.getId() == -1));// 输出数据
get,
packet.getType(),
body.length,
System.nanoTime() - start,
message.getId() == -1));// 输出数据
}
} else {
byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", get.toString()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(get.getDestination(),
get, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", get.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(get.getDestination(),
get,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
case CLIENTACK:
@@ -213,20 +252,38 @@ public class SessionHandler extends SimpleChannelHandler {
MDC.put("destination", ack.getDestination());
if (StringUtils.isNotEmpty(ack.getDestination()) && StringUtils.isNotEmpty(ack.getClientId())) {
if (ack.getBatchId() == 0L) {
byte[] errorBytes = NettyUtils.errorPacket(402, MessageFormatter.format("batchId should assign value", ack.toString()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ack.getDestination(),
ack, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 402));
byte[] errorBytes = NettyUtils.errorPacket(402,
MessageFormatter.format("batchId should assign value", ack.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(ack.getDestination(),
ack,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 402));
} else if (ack.getBatchId() == -1L) { // -1代表上一次get没有数据，直接忽略之
// donothing
} else {
clientIdentity = new ClientIdentity(ack.getDestination(), Short.valueOf(ack.getClientId()));
embeddedServer.ack(clientIdentity, ack.getBatchId());
new ChannelFutureAggregator(ack.getDestination(), ack, packet.getType(), 0, System.nanoTime() - start).operationComplete(null);
new ChannelFutureAggregator(ack.getDestination(),
ack,
packet.getType(),
0,
System.nanoTime() - start).operationComplete(null);
}
} else {
byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ack.getDestination(),
ack, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(ack.getDestination(),
ack,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
case CLIENTROLLBACK:
@@ -241,25 +298,41 @@ public class SessionHandler extends SimpleChannelHandler {
} else {
embeddedServer.rollback(clientIdentity, rollback.getBatchId()); // 只回滚单个批次
}
new ChannelFutureAggregator(rollback.getDestination(), rollback, packet.getType(), 0, System.nanoTime() - start).operationComplete(null);
new ChannelFutureAggregator(rollback.getDestination(),
rollback,
packet.getType(),
0,
System.nanoTime() - start).operationComplete(null);
} else {
byte[] errorBytes = NettyUtils.errorPacket(401, MessageFormatter.format("destination or clientId is null", rollback.toString()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(rollback.getDestination(),
rollback, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 401));
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", rollback.toString())
.getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(rollback.getDestination(),
rollback,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
default:
byte[] errorBytes = NettyUtils.errorPacket(400, MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel().getRemoteAddress().toString(),
null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
byte[] errorBytes = NettyUtils.errorPacket(400,
MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel()
.getRemoteAddress()
.toString(), null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
break;
}
} catch (Throwable exception) {
byte[] errorBytes = NettyUtils.errorPacket(400, MessageFormatter.format("something goes wrong with channel:{}, exception={}",
byte[] errorBytes = NettyUtils.errorPacket(400,
MessageFormatter.format("something goes wrong with channel:{}, exception={}",
ctx.getChannel(),
ExceptionUtils.getStackTrace(exception)).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel().getRemoteAddress().toString(),
null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel()
.getRemoteAddress()
.toString(), null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
} finally {
MDC.remove("destination");
}
server/src/test/java/com/alibaba/otter/canal/server/CanalServerTest.java
+13 -7
View File
@@ -34,6 +34,8 @@ import com.alibaba.otter.canal.protocol.CanalPacket.Sub;
import com.alibaba.otter.canal.protocol.CanalPacket.Unsub;
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
import com.alibaba.otter.canal.server.netty.NettyUtils;
@Ignore
public class CanalServerTest {
@@ -68,7 +70,6 @@ public class CanalServerTest {
@Test
public void testAuth() {
try {
SocketChannel channel = SocketChannel.open();
channel.connect(new InetSocketAddress("127.0.0.1", 1088));
@@ -93,6 +94,7 @@ public class CanalServerTest {
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.CLIENTAUTHENTICATION)
.setVersion(NettyUtils.VERSION)
.setBody(ca.toByteString())
.build()
.toByteArray());
@@ -107,12 +109,12 @@ public class CanalServerTest {
throw new Exception("something goes wrong when doing authentication: " + ack.getErrorMessage());
}
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.SUBSCRIPTION)
.setBody(Sub.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString())
.build()
.toByteArray());
writeWithHeader(channel, Packet.newBuilder()
.setType(PacketType.SUBSCRIPTION)
.setVersion(NettyUtils.VERSION)
.setBody(Sub.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString())
.build()
.toByteArray());
//
p = Packet.parseFrom(readNextPacket(channel));
ack = Ack.parseFrom(p.getBody());
@@ -124,6 +126,7 @@ public class CanalServerTest {
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.GET)
.setVersion(NettyUtils.VERSION)
.setBody(Get.newBuilder()
.setDestination(DESTINATION)
.setClientId("1")
@@ -158,6 +161,7 @@ public class CanalServerTest {
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.CLIENTACK)
.setVersion(NettyUtils.VERSION)
.setBody(ClientAck.newBuilder()
.setDestination(DESTINATION)
.setClientId("1")
@@ -171,6 +175,7 @@ public class CanalServerTest {
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.CLIENTROLLBACK)
.setVersion(NettyUtils.VERSION)
.setBody(ClientRollback.newBuilder()
.setDestination(DESTINATION)
.setClientId("1")
@@ -182,6 +187,7 @@ public class CanalServerTest {
writeWithHeader(channel,
Packet.newBuilder()
.setType(PacketType.UNSUBSCRIPTION)
.setVersion(NettyUtils.VERSION)
.setBody(Unsub.newBuilder().setDestination(DESTINATION).setClientId("1").build().toByteString())
.build()
.toByteArray());
server/src/test/java/com/alibaba/otter/canal/server/SecurityUtilTest.java
+22
View File
@@ -0,0 +1,22 @@
package com.alibaba.otter.canal.server;
import java.security.NoSuchAlgorithmException;
import org.junit.Assert;
import org.junit.Test;
import com.alibaba.otter.canal.protocol.SecurityUtil;
public class SecurityUtilTest {
@Test
public void testSimple() throws NoSuchAlgorithmException {
byte[] seed = { 1, 2, 3, 4, 5, 6, 7, 8 };
// String str = "e3619321c1a937c46a0d8bd1dac39f93b27d4458"; // canal
// passwd
String str = SecurityUtil.scrambleGenPass("canal".getBytes());
byte[] client = SecurityUtil.scramble411("canal".getBytes(), seed);
boolean check = SecurityUtil.scrambleServerAuth(client, SecurityUtil.hexStr2Bytes(str), seed);
Assert.assertTrue(check);
}
}