wyl/canal
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update guava to 18.0, magriate compute map by package level implement

Browse Source
This commit is contained in:
wener
2015-04-23 22:15:57 +08:00
parent bdf3b0c861
commit 6a0733bbab
14 changed files with 633 additions and 502 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java
+8 -5
View File
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.common.zookeeper;
import com.google.common.collect.MigrateMap;
import java.util.Map;
import org.I0Itec.zkclient.IZkConnection;
@@ -23,12 +24,14 @@ import com.google.common.collect.MapMaker;
public class ZkClientx extends ZkClient {
// 对于zkclient进行一次缓存，避免一个jvm内部使用多个zk connection
private static Map<String, ZkClientx> clients = new MapMaker().makeComputingMap(new Function<String, ZkClientx>() {
private static Map<String, ZkClientx> clients = MigrateMap.makeComputingMap(new Function<String, ZkClientx>()
{
public ZkClientx apply(String servers) {
return new ZkClientx(servers);
}
});
public ZkClientx apply(String servers)
{
return new ZkClientx(servers);
}
});
public static ZkClientx getZkClient(String servers) {
return clients.get(servers);
deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java
+63 -33
View File
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.deployer;
import com.google.common.collect.MigrateMap;
import java.util.Map;
import java.util.Properties;
@@ -70,9 +71,11 @@ public class CanalController {
}
public CanalController(final Properties properties){
managerClients = new MapMaker().makeComputingMap(new Function<String, CanalConfigClient>() {
managerClients = MigrateMap.makeComputingMap(new Function<String, CanalConfigClient>()
{
public CanalConfigClient apply(String managerAddress) {
public CanalConfigClient apply(String managerAddress)
{
return getManagerClient(managerAddress);
}
});
@@ -107,68 +110,88 @@ public class CanalController {
final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
ServerRunningMonitors.setServerData(serverData);
ServerRunningMonitors.setRunningMonitors(new MapMaker().makeComputingMap(new Function<String, ServerRunningMonitor>() {
public ServerRunningMonitor apply(final String destination) {
ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>()
{
public ServerRunningMonitor apply(final String destination)
{
ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
runningMonitor.setDestination(destination);
runningMonitor.setListener(new ServerRunningListener() {
runningMonitor.setListener(new ServerRunningListener()
{
public void processActiveEnter() {
try {
public void processActiveEnter()
{
try
{
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.start(destination);
} finally {
} finally
{
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processActiveExit() {
try {
public void processActiveExit()
{
try
{
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
embededCanalServer.stop(destination);
} finally {
} finally
{
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStart() {
try {
if (zkclientx != null) {
public void processStart()
{
try
{
if (zkclientx != null)
{
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
+ port);
+ port);
initCid(path);
zkclientx.subscribeStateChanges(new IZkStateListener() {
zkclientx.subscribeStateChanges(new IZkStateListener()
{
public void handleStateChanged(KeeperState state) throws Exception {
public void handleStateChanged(KeeperState state) throws Exception
{
}
public void handleNewSession() throws Exception {
public void handleNewSession() throws Exception
{
initCid(path);
}
});
}
} finally {
} finally
{
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
public void processStop() {
try {
public void processStop()
{
try
{
MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
if (zkclientx != null) {
if (zkclientx != null)
{
final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
+ port);
+ port);
releaseCid(path);
}
} finally {
} finally
{
MDC.remove(CanalConstants.MDC_DESTINATION);
}
}
});
if (zkclientx != null) {
if (zkclientx != null)
{
runningMonitor.setZkClient(zkclientx);
}
return runningMonitor;
@@ -216,25 +239,32 @@ public class CanalController {
}
};
instanceConfigMonitors = new MapMaker().makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>()
{
public InstanceConfigMonitor apply(InstanceMode mode) {
int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
public InstanceConfigMonitor apply(InstanceMode mode)
{
int scanInterval = Integer
.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
if (mode.isSpring()) {
if (mode.isSpring())
{
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
monitor.setScanIntervalInSecond(scanInterval);
monitor.setDefaultAction(defaultAction);
// 设置conf目录，默认是user.dir + conf目录组成
String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR);
if (StringUtils.isEmpty(rootDir)) {
if (StringUtils.isEmpty(rootDir))
{
rootDir = "../conf";
}
monitor.setRootConf(rootDir);
return monitor;
} else if (mode.isManager()) {
} else if (mode.isManager())
{
return new ManagerInstanceConfigMonitor();
} else {
} else
{
throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor");
}
}
deployer/src/main/java/com/alibaba/otter/canal/deployer/monitor/SpringInstanceConfigMonitor.java
+8 -5
View File
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.deployer.monitor;
import com.google.common.collect.MigrateMap;
import java.io.File;
import java.io.FileFilter;
import java.io.FilenameFilter;
@@ -39,12 +40,14 @@ public class SpringInstanceConfigMonitor extends AbstractCanalLifeCycle implemen
private long scanIntervalInSecond = 5;
private InstanceAction defaultAction = null;
private Map<String, InstanceAction> actions = new MapMaker().makeMap();
private Map<String, InstanceConfigFiles> lastFiles = new MapMaker().makeComputingMap(new Function<String, InstanceConfigFiles>() {
private Map<String, InstanceConfigFiles> lastFiles = MigrateMap.makeComputingMap(new Function<String, InstanceConfigFiles>()
{
public InstanceConfigFiles apply(String destination) {
return new InstanceConfigFiles(destination);
}
});
public InstanceConfigFiles apply(String destination)
{
return new InstanceConfigFiles(destination);
}
});
private ScheduledExecutorService executor = Executors.newScheduledThreadPool(1,
new NamedThreadFactory("canal-instance-scan"));
filter/src/main/java/com/alibaba/otter/canal/filter/PatternUtils.java
+32 -26
View File
@@ -1,48 +1,54 @@
package com.alibaba.otter.canal.filter;
import com.alibaba.otter.canal.filter.exception.CanalFilterException;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
import java.util.Map;
import org.apache.oro.text.regex.MalformedPatternException;
import org.apache.oro.text.regex.Pattern;
import org.apache.oro.text.regex.PatternCompiler;
import org.apache.oro.text.regex.Perl5Compiler;
import com.alibaba.otter.canal.filter.exception.CanalFilterException;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
/**
* 提供{@linkplain Pattern}的lazy get处理
*
*
* @author jianghang 2013-1-22 下午09:36:44
* @version 1.0.0
*/
public class PatternUtils {
public class PatternUtils
{
private static Map<String, Pattern> patterns = new MapMaker().softValues().makeComputingMap(
new Function<String, Pattern>() {
private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(new MapMaker().softValues(),
new Function<String, Pattern>()
{
public Pattern apply(
String pattern) {
try {
PatternCompiler pc = new Perl5Compiler();
return pc.compile(
pattern,
Perl5Compiler.CASE_INSENSITIVE_MASK
| Perl5Compiler.READ_ONLY_MASK
| Perl5Compiler.SINGLELINE_MASK);
} catch (MalformedPatternException e) {
throw new CanalFilterException(
e);
}
}
});
public Pattern apply(
String pattern)
{
try
{
PatternCompiler pc = new Perl5Compiler();
return pc.compile(
pattern,
Perl5Compiler.CASE_INSENSITIVE_MASK
| Perl5Compiler.READ_ONLY_MASK
| Perl5Compiler.SINGLELINE_MASK);
} catch (MalformedPatternException e)
{
throw new CanalFilterException(
e);
}
}
});
public static Pattern getPattern(String pattern) {
public static Pattern getPattern(String pattern)
{
return patterns.get(pattern);
}
public static void clear() {
public static void clear()
{
patterns.clear();
}
}
meta/src/main/java/com/alibaba/otter/canal/meta/FileMixedMetaManager.java
+17 -8
View File
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.meta;
import com.google.common.collect.MigrateMap;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -69,28 +70,36 @@ public class FileMixedMetaManager extends MemoryMetaManager implements CanalMeta
throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
}
dataFileCaches = new MapMaker().makeComputingMap(new Function<String, File>() {
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>()
{
public File apply(String destination) {
public File apply(String destination)
{
return getDataFile(destination);
}
});
executor = Executors.newScheduledThreadPool(1);
destinations = new MapMaker().makeComputingMap(new Function<String, List<ClientIdentity>>() {
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
{
public List<ClientIdentity> apply(String destination) {
public List<ClientIdentity> apply(String destination)
{
return loadClientIdentity(destination);
}
});
cursors = new MapMaker().makeComputingMap(new Function<ClientIdentity, Position>() {
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>()
{
public Position apply(ClientIdentity clientIdentity) {
public Position apply(ClientIdentity clientIdentity)
{
Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
if (position == null) {
if (position == null)
{
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
} else
{
return position;
}
}
meta/src/main/java/com/alibaba/otter/canal/meta/MemoryMetaManager.java
+9 -4
View File
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.meta;
import com.google.common.collect.MigrateMap;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -31,9 +32,11 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
public void start() {
super.start();
batches = new MapMaker().makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>()
{
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity)
{
return MemoryClientIdentityBatch.create(clientIdentity);
}
@@ -41,9 +44,11 @@ public class MemoryMetaManager extends AbstractCanalLifeCycle implements CanalMe
cursors = new MapMaker().makeMap();
destinations = new MapMaker().makeComputingMap(new Function<String, List<ClientIdentity>>() {
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
{
public List<ClientIdentity> apply(String destination) {
public List<ClientIdentity> apply(String destination)
{
return Lists.newArrayList();
}
});
meta/src/main/java/com/alibaba/otter/canal/meta/MixedMetaManager.java
+89 -51
View File
@@ -1,68 +1,79 @@
package com.alibaba.otter.canal.meta;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.springframework.util.Assert;
import com.alibaba.otter.canal.meta.exception.CanalMetaManagerException;
import com.alibaba.otter.canal.protocol.ClientIdentity;
import com.alibaba.otter.canal.protocol.position.Position;
import com.alibaba.otter.canal.protocol.position.PositionRange;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import com.google.common.collect.MigrateMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.springframework.util.Assert;
/**
* 组合memory + zookeeper的使用模式
*
*
* @author jianghang 2012-7-11 下午03:58:00
* @version 1.0.0
*/
public class MixedMetaManager extends MemoryMetaManager implements CanalMetaManager {
public class MixedMetaManager extends MemoryMetaManager implements CanalMetaManager
{
private ExecutorService executor;
private ExecutorService executor;
private ZooKeeperMetaManager zooKeeperMetaManager;
@SuppressWarnings("serial")
private final Position nullCursor = new Position() {
};
private final Position nullCursor = new Position()
{
};
public void start() {
public void start()
{
super.start();
Assert.notNull(zooKeeperMetaManager);
if (!zooKeeperMetaManager.isStart()) {
if (!zooKeeperMetaManager.isStart())
{
zooKeeperMetaManager.start();
}
executor = Executors.newFixedThreadPool(1);
destinations = new MapMaker().makeComputingMap(new Function<String, List<ClientIdentity>>() {
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
{
public List<ClientIdentity> apply(String destination) {
public List<ClientIdentity> apply(String destination)
{
return zooKeeperMetaManager.listAllSubscribeInfo(destination);
}
});
cursors = new MapMaker().makeComputingMap(new Function<ClientIdentity, Position>() {
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>()
{
public Position apply(ClientIdentity clientIdentity) {
public Position apply(ClientIdentity clientIdentity)
{
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null) {
if (position == null)
{
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
} else
{
return position;
}
}
});
batches = new MapMaker().makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>()
{
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity)
{
// 读取一下zookeeper信息，初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet())
{
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;
@@ -70,10 +81,12 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
});
}
public void stop() {
public void stop()
{
super.stop();
if (zooKeeperMetaManager.isStart()) {
if (zooKeeperMetaManager.isStart())
{
zooKeeperMetaManager.stop();
}
@@ -82,58 +95,73 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
batches.clear();
}
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException
{
super.subscribe(clientIdentity);
executor.submit(new Runnable() {
executor.submit(new Runnable()
{
public void run() {
public void run()
{
zooKeeperMetaManager.subscribe(clientIdentity);
}
});
}
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException
{
super.unsubscribe(clientIdentity);
executor.submit(new Runnable() {
executor.submit(new Runnable()
{
public void run() {
public void run()
{
zooKeeperMetaManager.unsubscribe(clientIdentity);
}
});
}
public void updateCursor(final ClientIdentity clientIdentity, final Position position)
throws CanalMetaManagerException {
throws CanalMetaManagerException
{
super.updateCursor(clientIdentity, position);
// 异步刷新
executor.submit(new Runnable() {
executor.submit(new Runnable()
{
public void run() {
public void run()
{
zooKeeperMetaManager.updateCursor(clientIdentity, position);
}
});
}
@Override
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException
{
Position position = super.getCursor(clientIdentity);
if (position == nullCursor) {
if (position == nullCursor)
{
return null;
} else {
} else
{
return position;
}
}
public Long addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange)
throws CanalMetaManagerException {
throws CanalMetaManagerException
{
final Long batchId = super.addBatch(clientIdentity, positionRange);
// 异步刷新
executor.submit(new Runnable() {
executor.submit(new Runnable()
{
public void run() {
public void run()
{
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
}
});
@@ -141,24 +169,30 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
}
public void addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange, final Long batchId)
throws CanalMetaManagerException {
throws CanalMetaManagerException
{
super.addBatch(clientIdentity, positionRange, batchId);
// 异步刷新
executor.submit(new Runnable() {
executor.submit(new Runnable()
{
public void run() {
public void run()
{
zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
}
});
}
public PositionRange removeBatch(final ClientIdentity clientIdentity, final Long batchId)
throws CanalMetaManagerException {
throws CanalMetaManagerException
{
PositionRange positionRange = super.removeBatch(clientIdentity, batchId);
// 异步刷新
executor.submit(new Runnable() {
executor.submit(new Runnable()
{
public void run() {
public void run()
{
zooKeeperMetaManager.removeBatch(clientIdentity, batchId);
}
});
@@ -166,20 +200,24 @@ public class MixedMetaManager extends MemoryMetaManager implements CanalMetaMana
return positionRange;
}
public void clearAllBatchs(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
public void clearAllBatchs(final ClientIdentity clientIdentity) throws CanalMetaManagerException
{
super.clearAllBatchs(clientIdentity);
// 异步刷新
executor.submit(new Runnable() {
executor.submit(new Runnable()
{
public void run() {
public void run()
{
zooKeeperMetaManager.clearAllBatchs(clientIdentity);
}
});
}
// =============== setter / getter ================
public void setZooKeeperMetaManager(ZooKeeperMetaManager zooKeeperMetaManager) {
public void setZooKeeperMetaManager(ZooKeeperMetaManager zooKeeperMetaManager)
{
this.zooKeeperMetaManager = zooKeeperMetaManager;
}
}
meta/src/main/java/com/alibaba/otter/canal/meta/PeriodMixedMetaManager.java
+19 -10
View File
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.meta;
import com.google.common.collect.MigrateMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -52,32 +53,40 @@ public class PeriodMixedMetaManager extends MemoryMetaManager implements CanalMe
}
executor = Executors.newScheduledThreadPool(1);
destinations = new MapMaker().makeComputingMap(new Function<String, List<ClientIdentity>>() {
destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>()
{
public List<ClientIdentity> apply(String destination) {
public List<ClientIdentity> apply(String destination)
{
return zooKeeperMetaManager.listAllSubscribeInfo(destination);
}
});
cursors = new MapMaker().makeComputingMap(new Function<ClientIdentity, Position>() {
cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>()
{
public Position apply(ClientIdentity clientIdentity) {
public Position apply(ClientIdentity clientIdentity)
{
Position position = zooKeeperMetaManager.getCursor(clientIdentity);
if (position == null) {
if (position == null)
{
return nullCursor; // 返回一个空对象标识，避免出现异常
} else {
} else
{
return position;
}
}
});
batches = new MapMaker().makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>()
{
public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity)
{
// 读取一下zookeeper信息，初始化一次
MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet())
{
batches.addPositionRange(entry.getValue(), entry.getKey()); // 添加记录到指定batchId
}
return batches;
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
+13 -6
View File
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.parse.inbound.mysql.dbsync;
import com.google.common.collect.MigrateMap;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -38,17 +39,23 @@ public class TableMetaCache {
public TableMetaCache(MysqlConnection con){
this.connection = con;
tableMetaCache = new MapMaker().makeComputingMap(new Function<String, TableMeta>() {
tableMetaCache = MigrateMap.makeComputingMap(new Function<String, TableMeta>()
{
public TableMeta apply(String name) {
try {
public TableMeta apply(String name)
{
try
{
return getTableMeta0(name);
} catch (IOException e) {
} catch (IOException e)
{
// 尝试做一次retry操作
try {
try
{
connection.reconnect();
return getTableMeta0(name);
} catch (IOException e1) {
} catch (IOException e1)
{
throw new CanalParseException("fetch failed by table meta:" + name, e1);
}
}
parse/src/main/java/com/alibaba/otter/canal/parse/index/FileMixedLogPositionManager.java
+13 -6
View File
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.parse.index;
import com.google.common.collect.MigrateMap;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
@@ -67,21 +68,27 @@ public class FileMixedLogPositionManager extends MemoryLogPositionManager {
throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
}
dataFileCaches = new MapMaker().makeComputingMap(new Function<String, File>() {
dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>()
{
public File apply(String destination) {
public File apply(String destination)
{
return getDataFile(destination);
}
});
executor = Executors.newScheduledThreadPool(1);
positions = new MapMaker().makeComputingMap(new Function<String, LogPosition>() {
positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>()
{
public LogPosition apply(String destination) {
public LogPosition apply(String destination)
{
LogPosition logPosition = loadDataFromFile(dataFileCaches.get(destination));
if (logPosition == null) {
if (logPosition == null)
{
return nullPosition;
} else {
} else
{
return logPosition;
}
}
parse/src/main/java/com/alibaba/otter/canal/parse/index/MixedLogPositionManager.java
+9 -4
View File
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.parse.index;
import com.google.common.collect.MigrateMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -34,13 +35,17 @@ public class MixedLogPositionManager extends MemoryLogPositionManager implements
zooKeeperLogPositionManager.start();
}
executor = Executors.newFixedThreadPool(1);
positions = new MapMaker().makeComputingMap(new Function<String, LogPosition>() {
positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>()
{
public LogPosition apply(String destination) {
public LogPosition apply(String destination)
{
LogPosition logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
if (logPosition == null) {
if (logPosition == null)
{
return nullPosition;
} else {
} else
{
return logPosition;
}
}
parse/src/main/java/com/alibaba/otter/canal/parse/index/PeriodMixedLogPositionManager.java
+9 -4
View File
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.parse.index;
import com.google.common.collect.MigrateMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@@ -43,13 +44,17 @@ public class PeriodMixedLogPositionManager extends MemoryLogPositionManager impl
zooKeeperLogPositionManager.start();
}
executor = Executors.newScheduledThreadPool(1);
positions = new MapMaker().makeComputingMap(new Function<String, LogPosition>() {
positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>()
{
public LogPosition apply(String destination) {
public LogPosition apply(String destination)
{
LogPosition logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
if (logPosition == null) {
if (logPosition == null)
{
return nullPosition;
} else {
} else
{
return logPosition;
}
}
pom.xml
+339 -338
View File
@@ -1,311 +1,312 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<packaging>pom</packaging>
<name>canal module for otter ${project.version}</name>
<version>1.0.20-SNAPSHOT</version>
<url>https://github.com/alibaba/canal</url>
<parent>
<groupId>org.sonatype.oss</groupId>
<artifactId>oss-parent</artifactId>
<version>7</version>
</parent>
<developers>
<developer>
<name>agapple</name>
<url>http://agapple.iteye.com</url>
<email>jianghang115@gmail.com</email>
<timezone>8</timezone>
</developer>
<developer>
<name>zavakid</name>
<url>http://www.zavakid.com</url>
<email>zava.kid@gmail.com</email>
<timezone>8</timezone>
</developer>
<developer>
<name>in355hz</name>
<url>http://in355hz.iteye.com</url>
<email>in355hz@gmail.com</email>
<timezone>8</timezone>
</developer>
</developers>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.alibaba.otter</groupId>
<artifactId>canal</artifactId>
<packaging>pom</packaging>
<name>canal module for otter ${project.version}</name>
<version>1.0.20-SNAPSHOT</version>
<url>https://github.com/alibaba/canal</url>
<parent>
<groupId>org.sonatype.oss</groupId>
<artifactId>oss-parent</artifactId>
<version>7</version>
</parent>
<developers>
<developer>
<name>agapple</name>
<url>http://agapple.iteye.com</url>
<email>jianghang115@gmail.com</email>
<timezone>8</timezone>
</developer>
<developer>
<name>zavakid</name>
<url>http://www.zavakid.com</url>
<email>zava.kid@gmail.com</email>
<timezone>8</timezone>
</developer>
<developer>
<name>in355hz</name>
<url>http://in355hz.iteye.com</url>
<email>in355hz@gmail.com</email>
<timezone>8</timezone>
</developer>
</developers>
<licenses>
<license>
<name>Apache License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0</url>
</license>
</licenses>
<licenses>
<license>
<name>Apache License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0</url>
</license>
</licenses>
<scm>
<url>git@github.com:alibaba/canal.git</url>
<connection>scm:git:git@github.com:alibaba/canal.git</connection>
<developerConnection>scm:git:git@github.com:alibaba/canal.git</developerConnection>
</scm>
<repositories>
<repository>
<id>central</id>
<url>http://repo1.maven.org/maven2</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>java.net</id>
<url>http://download.java.net/maven/2/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>alibaba</id>
<url>http://code.alibabatech.com/mvn/releases/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>sonatype</id>
<name>sonatype</name>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
<repository>
<id>sonatype-release</id>
<name>sonatype-release</name>
<url>https://oss.sonatype.org/service/local/repositories/releases/content</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!--maven properties-->
<maven.test.skip>true</maven.test.skip>
<downloadSources>true</downloadSources>
<!-- compiler settings properties -->
<java_source_version>1.6</java_source_version>
<java_target_version>1.6</java_target_version>
<file_encoding>UTF-8</file_encoding>
</properties>
<modules>
<module>common</module>
<module>meta</module>
<module>dbsync</module>
<module>filter</module>
<module>driver</module>
<module>parse</module>
<module>sink</module>
<module>store</module>
<module>protocol</module>
<module>instance</module>
<module>server</module>
<module>client</module>
<module>deployer</module>
<module>example</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring</artifactId>
<version>2.5.6</version>
</dependency>
<scm>
<url>git@github.com:alibaba/canal.git</url>
<connection>scm:git:git@github.com:alibaba/canal.git</connection>
<developerConnection>scm:git:git@github.com:alibaba/canal.git</developerConnection>
</scm>
<repositories>
<repository>
<id>central</id>
<url>http://repo1.maven.org/maven2</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>java.net</id>
<url>http://download.java.net/maven/2/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>alibaba</id>
<url>http://code.alibabatech.com/mvn/releases/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>sonatype</id>
<name>sonatype</name>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
<repository>
<id>sonatype-release</id>
<name>sonatype-release</name>
<url>https://oss.sonatype.org/service/local/repositories/releases/content</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!--maven properties-->
<maven.test.skip>true</maven.test.skip>
<downloadSources>true</downloadSources>
<!-- compiler settings properties -->
<java_source_version>1.6</java_source_version>
<java_target_version>1.6</java_target_version>
<file_encoding>UTF-8</file_encoding>
</properties>
<modules>
<module>common</module>
<module>meta</module>
<module>dbsync</module>
<module>filter</module>
<module>driver</module>
<module>parse</module>
<module>sink</module>
<module>store</module>
<module>protocol</module>
<module>instance</module>
<module>server</module>
<module>client</module>
<module>deployer</module>
<module>example</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring</artifactId>
<version>2.5.6</version>
</dependency>
<!-- external -->
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.github.sgroschupf</groupId>
<artifactId>zkclient</artifactId>
<version>0.1</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.1.26</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>r09</version>
</dependency>
<dependency>
<groupId>com.googlecode.aviator</groupId>
<artifactId>aviator</artifactId>
<version>2.2.1</version>
</dependency>
<dependency>
<groupId>oro</groupId>
<artifactId>oro</artifactId>
<version>2.0.8</version>
</dependency>
<dependency>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
<version>3.2.5.Final</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>2.4.1</version>
</dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.6</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.5</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.github.sgroschupf</groupId>
<artifactId>zkclient</artifactId>
<version>0.1</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.1.26</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>18.0</version>
</dependency>
<dependency>
<groupId>com.googlecode.aviator</groupId>
<artifactId>aviator</artifactId>
<version>2.2.1</version>
</dependency>
<dependency>
<groupId>oro</groupId>
<artifactId>oro</artifactId>
<version>2.0.8</version>
</dependency>
<dependency>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
<version>3.2.5.Final</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>2.4.1</version>
</dependency>
<!-- log -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>1.0.6</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.0.6</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
<version>1.6.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.6.0</version>
</dependency>
<!-- test dependency -->
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>1.1.3</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.12</version>
<scope>test</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<extensions>
<extension>
<groupId>org.jvnet.wagon-svn</groupId>
<artifactId>wagon-svn</artifactId>
<version>1.9</version>
</extension>
<extension>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-http-shared</artifactId>
<version>1.0-beta-7</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.1.3</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jcl-over-slf4j</artifactId>
<version>1.7.12</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.12</version>
</dependency>
<!-- test dependency -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.12</version>
<scope>test</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<extensions>
<extension>
<groupId>org.jvnet.wagon-svn</groupId>
<artifactId>wagon-svn</artifactId>
<version>1.9</version>
</extension>
<extension>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-http-shared</artifactId>
<version>1.0-beta-7</version>
</extension>
</extensions>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.4</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.2</version>
<configuration>
<source>${java_source_version}</source>
<target>${java_target_version}</target>
<encoding>${file_encoding}</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-eclipse-plugin</artifactId>
<version>2.5.1</version>
<configuration>
<additionalConfig>
<file>
<name>.settings/org.eclipse.core.resources.prefs</name>
<content>
<![CDATA[eclipse.preferences.version=1${line.separator}encoding/<project>=${file_encoding}${line.separator}]]>
</content>
</file>
</additionalConfig>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.5</version>
<configuration>
<includes>
<include>**/*Test.java</include>
</includes>
<excludes>
<exclude>**/*NoRunTest.java</exclude>
</excludes>
</configuration>
</plugin>
<configuration>
<source>${java_source_version}</source>
<target>${java_target_version}</target>
<encoding>${file_encoding}</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-eclipse-plugin</artifactId>
<version>2.5.1</version>
<configuration>
<additionalConfig>
<file>
<name>.settings/org.eclipse.core.resources.prefs</name>
<content>
<![CDATA[eclipse.preferences.version=1${line.separator}encoding/<project>=${file_encoding}${line.separator}]]>
</content>
</file>
</additionalConfig>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.5</version>
<configuration>
<includes>
<include>**/*Test.java</include>
</includes>
<excludes>
<exclude>**/*NoRunTest.java</exclude>
</excludes>
</configuration>
</plugin>
<!-- javadoc -->
<!--
<plugin>
@@ -362,43 +363,43 @@
</executions>
</plugin>
-->
</plugins>
<sourceDirectory>src/main/java</sourceDirectory>
<testSourceDirectory>src/test/java</testSourceDirectory>
<resources>
<resource>
<directory>src/main/resources</directory>
<includes>
<include>**/*</include>
</includes>
<excludes>
<exclude>**/.svn/</exclude>
</excludes>
</resource>
</resources>
<testResources>
<testResource>
<directory>src/test/resources</directory>
<includes>
<include>**/*</include>
</includes>
<excludes>
<exclude>**/.svn/</exclude>
</excludes>
</testResource>
</testResources>
</build>
<distributionManagement>
<snapshotRepository>
<id>sonatype-nexus-snapshots</id>
<name>Sonatype Nexus Snapshots</name>
<url>https://oss.sonatype.org/content/repositories/snapshots/</url>
</snapshotRepository>
<repository>
<id>sonatype-nexus-staging</id>
<name>Nexus Release Repository</name>
<url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>
</plugins>
<sourceDirectory>src/main/java</sourceDirectory>
<testSourceDirectory>src/test/java</testSourceDirectory>
<resources>
<resource>
<directory>src/main/resources</directory>
<includes>
<include>**/*</include>
</includes>
<excludes>
<exclude>**/.svn/</exclude>
</excludes>
</resource>
</resources>
<testResources>
<testResource>
<directory>src/test/resources</directory>
<includes>
<include>**/*</include>
</includes>
<excludes>
<exclude>**/.svn/</exclude>
</excludes>
</testResource>
</testResources>
</build>
<distributionManagement>
<snapshotRepository>
<id>sonatype-nexus-snapshots</id>
<name>Sonatype Nexus Snapshots</name>
<url>https://oss.sonatype.org/content/repositories/snapshots/</url>
</snapshotRepository>
<repository>
<id>sonatype-nexus-staging</id>
<name>Nexus Release Repository</name>
<url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>
</project>
server/src/main/java/com/alibaba/otter/canal/server/embedded/CanalServerWithEmbedded.java
+5 -2
View File
@@ -1,5 +1,6 @@
package com.alibaba.otter.canal.server.embedded;
import com.google.common.collect.MigrateMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -47,9 +48,11 @@ public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements C
public void start() {
super.start();
canalInstances = new MapMaker().makeComputingMap(new Function<String, CanalInstance>() {
canalInstances = MigrateMap.makeComputingMap(new Function<String, CanalInstance>()
{
public CanalInstance apply(String destination) {
public CanalInstance apply(String destination)
{
return canalInstanceGenerator.generate(destination);
}
});