zookeeper的Leader选举源码解析

Tech

导读

zookeeper是一个分布式服务框架，主要解决分布式应用中常见的多种数据问题，例如集群管理，状态同步等。为解决这些问题zookeeper需要Leader选举进行保障数据的强一致性机制和稳定性。本文通过集群的配置，对leader选举源进行解析，让读者们了解如何利用BIO通信机制，多线程多层队列实现高性能架构。

       每一个zookeeper服务端称之为一个节点，每个节点都有投票权，把其选票投向每一个有选举权的节点，当其中一个节点选举出票数过半，这个节点就会成为Leader，其它节点成为Follower。

【plain】zoo1.cfg文件内容：dataDir=/export/data/zookeeper-1clientPort=2181server.1=127.0.0.1:2001:3001server.2=127.0.0.1:2002:3002:participantserver.3=127.0.0.1:2003:3003:participantserver.4=127.0.0.1:2004:3004:observer
zoo2.cfg文件内容：dataDir=/export/data/zookeeper-2clientPort=2182server.1=127.0.0.1:2001:3001server.2=127.0.0.1:2002:3002:participantserver.3=127.0.0.1:2003:3003:participantserver.4=127.0.0.1:2004:3004:observer
zoo3.cfg文件内容：dataDir=/export/data/zookeeper-3clientPort=2183server.1=127.0.0.1:2001:3001server.2=127.0.0.1:2002:3002:participantserver.3=127.0.0.1:2003:3003:participantserver.4=127.0.0.1:2004:3004:observer
zoo4.cfg文件内容：dataDir=/export/data/zookeeper-4clientPort=2184server.1=127.0.0.1:2001:3001server.2=127.0.0.1:2002:3002:participantserver.3=127.0.0.1:2003:3003:participantserver.4=127.0.0.1:2004:3004:observer

【Java】public QuorumMaj(Properties props) throws ConfigException {        for (Entry<Object, Object> entry : props.entrySet()) {            String key = entry.getKey().toString();            String value = entry.getValue().toString();            //读取集群配置文件中的server.开头的应用实例配置信息            if (key.startsWith("server.")) {                int dot = key.indexOf('.');                long sid = Long.parseLong(key.substring(dot + 1));                QuorumServer qs = new QuorumServer(sid, value);                allMembers.put(Long.valueOf(sid), qs);                if (qs.type == LearnerType.PARTICIPANT)//应用实例绑定的角色为PARTICIPANT意为参与选举                    votingMembers.put(Long.valueOf(sid), qs);                else {                    //观察者成员                    observingMembers.put(Long.valueOf(sid), qs);                }            } else if (key.equals("version")) {                version = Long.parseLong(value, 16);            }        }        //过半基数        half = votingMembers.size() / 2;    }

【plain】synchronized public void startLeaderElection(){try {           if (getPeerState() == ServerState.LOOKING) {               //首轮：当前节点默认投票对象为自己               currentVote = new Vote(myid, getLastLoggedZxid(), getCurrentEpoch());           }       } catch(IOException e) {           RuntimeException re = new RuntimeException(e.getMessage());           re.setStackTrace(e.getStackTrace());           throw re;       }//........}

【Java】//QuorumPeer.createCnxnManager()public QuorumCnxManager(QuorumPeer self,                        final long mySid,                        Map<Long,QuorumPeer.QuorumServer> view,                        QuorumAuthServer authServer,                        QuorumAuthLearner authLearner,                        int socketTimeout,                        boolean listenOnAllIPs,                        int quorumCnxnThreadsSize,                        boolean quorumSaslAuthEnabled) {    //接收投票队列(第二层传输队列)    this.recvQueue = new ArrayBlockingQueue<Message>(RECV_CAPACITY);    //按sid发送投票队列(第二层传输队列)    this.queueSendMap = new ConcurrentHashMap<Long, ArrayBlockingQueue<ByteBuffer>>();    //发送投票工作线程容器，表示着与sid投票节点已连接     this.senderWorkerMap = new ConcurrentHashMap<Long, SendWorker>();    this.lastMessageSent = new ConcurrentHashMap<Long, ByteBuffer>();
    String cnxToValue = System.getProperty("zookeeper.cnxTimeout");    if(cnxToValue != null){        this.cnxTO = Integer.parseInt(cnxToValue);    }
    this.self = self;
    this.mySid = mySid;    this.socketTimeout = socketTimeout;    this.view = view;    this.listenOnAllIPs = listenOnAllIPs;
    initializeAuth(mySid, authServer, authLearner, quorumCnxnThreadsSize,            quorumSaslAuthEnabled);    // Starts listener thread that waits for connection requests     //创建选举监听线程 接收选举投票请求    listener = new Listener();    listener.setName("QuorumPeerListener");}//QuorumPeer.createElectionAlgorithmprotected Election createElectionAlgorithm(int electionAlgorithm){    Election le=null;    //TODO: use a factory rather than a switch    switch (electionAlgorithm) {    case 0:        le = new LeaderElection(this);        break;    case 1:        le = new AuthFastLeaderElection(this);        break;    case 2:        le = new AuthFastLeaderElection(this, true);        break;    case 3:        qcm = createCnxnManager();// new QuorumCnxManager(... new Listener())        QuorumCnxManager.Listener listener = qcm.listener;        if(listener != null){            listener.start();//启动选举监听线程            FastLeaderElection fle = new FastLeaderElection(this, qcm);            fle.start();            le = fle;        } else {            LOG.error("Null listener when initializing cnx manager");        }        break;    default:        assert false;    }return le;}

【Java】//上面的listener.start()执行后，选择此方法public void run() {    int numRetries = 0;    InetSocketAddress addr;    Socket client = null;    while((!shutdown) && (numRetries < 3)){        try {            ss = new ServerSocket();            ss.setReuseAddress(true);            if (self.getQuorumListenOnAllIPs()) {                int port = self.getElectionAddress().getPort();                addr = new InetSocketAddress(port);            } else {                // Resolve hostname for this server in case the                // underlying ip address has changed.                self.recreateSocketAddresses(self.getId());                addr = self.getElectionAddress();            }            LOG.info("My election bind port: " + addr.toString());            setName(addr.toString());            ss.bind(addr);            while (!shutdown) {                client = ss.accept();                setSockOpts(client);                LOG.info("Received connection request "                        + client.getRemoteSocketAddress());                // Receive and handle the connection request                // asynchronously if the quorum sasl authentication is                // enabled. This is required because sasl server                // authentication process may take few seconds to finish,                // this may delay next peer connection requests.                if (quorumSaslAuthEnabled) {                    receiveConnectionAsync(client);                } else {//接收连接信息                    receiveConnection(client);                }                numRetries = 0;            }        } catch (IOException e) {            if (shutdown) {                break;            }            LOG.error("Exception while listening", e);            numRetries++;            try {                ss.close();                Thread.sleep(1000);            } catch (IOException ie) {                LOG.error("Error closing server socket", ie);            } catch (InterruptedException ie) {                LOG.error("Interrupted while sleeping. " +                    "Ignoring exception", ie);            }            closeSocket(client);        }    }    LOG.info("Leaving listener");    if (!shutdown) {        LOG.error("As I'm leaving the listener thread, "                + "I won't be able to participate in leader "                + "election any longer: "                + self.getElectionAddress());    } else if (ss != null) {        // Clean up for shutdown.        try {            ss.close();        } catch (IOException ie) {            // Don't log an error for shutdown.            LOG.debug("Error closing server socket", ie);        }    }}
//代码执行路径：receiveConnection()->handleConnection(...)private void handleConnection(Socket sock, DataInputStream din)            throws IOException {//...省略     if (sid < self.getId()) {            /*             * This replica might still believe that the connection to sid is             * up, so we have to shut down the workers before trying to open a             * new connection.             */            SendWorker sw = senderWorkerMap.get(sid);            if (sw != null) {                sw.finish();            }
            /*             * Now we start a new connection             */            LOG.debug("Create new connection to server: {}", sid);            closeSocket(sock);
            if (electionAddr != null) {                connectOne(sid, electionAddr);            } else {                connectOne(sid);            }
        } else { // Otherwise start worker threads to receive data.            SendWorker sw = new SendWorker(sock, sid);            RecvWorker rw = new RecvWorker(sock, din, sid, sw);            sw.setRecv(rw);
            SendWorker vsw = senderWorkerMap.get(sid);
            if (vsw != null) {                vsw.finish();            }  //存储连接信息<sid,SendWorker>            senderWorkerMap.put(sid, sw);
            queueSendMap.putIfAbsent(sid,                    new ArrayBlockingQueue<ByteBuffer>(SEND_CAPACITY));
            sw.start();            rw.start();     }}

【Java】//FastLeaderElection.starterprivate void starter(QuorumPeer self, QuorumCnxManager manager) {    this.self = self;    proposedLeader = -1;    proposedZxid = -1;    //发送队列sendqueue（第一层队列）    sendqueue = new LinkedBlockingQueue<ToSend>();    //接收队列recvqueue（第一层队列）    recvqueue = new LinkedBlockingQueue<Notification>();    this.messenger = new Messenger(manager);}//new Messenger(manager)Messenger(QuorumCnxManager manager) {    //创建线程WorkerSender    this.ws = new WorkerSender(manager);
    this.wsThread = new Thread(this.ws,            "WorkerSender[myid=" + self.getId() + "]");    this.wsThread.setDaemon(true);    //创建线程WorkerReceiver    this.wr = new WorkerReceiver(manager);
    this.wrThread = new Thread(this.wr,            "WorkerReceiver[myid=" + self.getId() + "]");    this.wrThread.setDaemon(true)；}

【Java】//FastLeaderElection.Messenger.WorkerSenderclass WorkerSender extends ZooKeeperThread{//...  public void run() {    while (!stop) {        try {            ToSend m = sendqueue.poll(3000, TimeUnit.MILLISECONDS);            if(m == null) continue;  //将投票信息发送出去            process(m);        } catch (InterruptedException e) {            break;        }    }    LOG.info("WorkerSender is down");  }}//QuorumCnxManager#toSendpublic void toSend(Long sid, ByteBuffer b) {    /*     * If sending message to myself, then simply enqueue it (loopback).     */    if (this.mySid == sid) {         b.position(0);         addToRecvQueue(new Message(b.duplicate(), sid));        /*         * Otherwise send to the corresponding thread to send.         */    } else {         /*          * Start a new connection if doesn't have one already.          */         ArrayBlockingQueue<ByteBuffer> bq = new ArrayBlockingQueue<ByteBuffer>(            SEND_CAPACITY);         ArrayBlockingQueue<ByteBuffer> oldq = queueSendMap.putIfAbsent(sid, bq);         //转储到queueSendMap<sid,queue>第二层传输队列中         if (oldq != null) {             addToSendQueue(oldq, b);         } else {             addToSendQueue(bq, b);         }         connectOne(sid);         }}

【Java】//WorkerReceiverpublic void run() {    Message response;    while (!stop) {      // Sleeps on receive      try {          //自旋获取recvQueue第二层传输队列元素          response = manager.pollRecvQueue(3000, TimeUnit.MILLISECONDS);          if(response == null) continue;          // The current protocol and two previous generations all send at least 28 bytes          if (response.buffer.capacity() < 28) {              LOG.error("Got a short response: " + response.buffer.capacity());              continue;          }          //...  if(self.getPeerState() == QuorumPeer.ServerState.LOOKING){         //第二层传输队列元素转存到recvqueue第一层队列中         recvqueue.offer(n);         //...      }    }//...}

【plain】//QuorunPeer.run//...try {   reconfigFlagClear();    if (shuttingDownLE) {       shuttingDownLE = false;       startLeaderElection();       }    //makeLEStrategy().lookForLeader() 发送投票    setCurrentVote(makeLEStrategy().lookForLeader());} catch (Exception e) {    LOG.warn("Unexpected exception", e);    setPeerState(ServerState.LOOKING);}  //...//FastLeaderElection.lookLeaderpublic Vote lookForLeader() throws InterruptedException {//...  //向其他应用发送投票sendNotifications();//...}
private void sendNotifications() {    //获取应用节点    for (long sid : self.getCurrentAndNextConfigVoters()) {        QuorumVerifier qv = self.getQuorumVerifier();        ToSend notmsg = new ToSend(ToSend.mType.notification,                proposedLeader,                proposedZxid,                logicalclock.get(),                QuorumPeer.ServerState.LOOKING,                sid,                proposedEpoch, qv.toString().getBytes());        if(LOG.isDebugEnabled()){            LOG.debug("Sending Notification: " + proposedLeader + " (n.leader), 0x"  +                  Long.toHexString(proposedZxid) + " (n.zxid), 0x" + Long.toHexString(logicalclock.get())  +                  " (n.round), " + sid + " (recipient), " + self.getId() +                  " (myid), 0x" + Long.toHexString(proposedEpoch) + " (n.peerEpoch)");        }        //储存投票信息        sendqueue.offer(notmsg);    }}
class WorkerSender extends ZooKeeperThread {    //...    public void run() {    while (!stop) {        try {//提取已储存的投票信息            ToSend m = sendqueue.poll(3000, TimeUnit.MILLISECONDS);            if(m == null) continue;
            process(m);        } catch (InterruptedException e) {            break;        }    }    LOG.info("WorkerSender is down");  }//...}

【Java】public Vote lookForLeader() throws InterruptedException {//.../* * Loop in which we exchange notifications until we find a leader */while ((self.getPeerState() == ServerState.LOOKING) &&        (!stop)){    /*     * Remove next notification from queue, times out after 2 times     * the termination time     */    //提取投递过来的选票信息    Notification n = recvqueue.poll(notTimeout,            TimeUnit.MILLISECONDS);/* * Sends more notifications if haven't received enough. * Otherwise processes new notification. */if(n == null){    if(manager.haveDelivered()){        //已全部连接成功，并且前一轮投票都完成，需要再次发起投票        sendNotifications();    } else {        //如果未收到选票信息，manager.contentAll()自动连接其它socket节点        manager.connectAll();    }    /*     * Exponential backoff     */    int tmpTimeOut = notTimeout*2;    notTimeout = (tmpTimeOut < maxNotificationInterval?            tmpTimeOut : maxNotificationInterval);    LOG.info("Notification time out: " + notTimeout);         }     //....    }  //...}

【Java】//manager.connectAll()->connectOne(sid)->initiateConnection(...)->startConnection(...)
private boolean startConnection(Socket sock, Long sid)        throws IOException {    DataOutputStream dout = null;    DataInputStream din = null;    try {        // Use BufferedOutputStream to reduce the number of IP packets. This is        // important for x-DC scenarios.        BufferedOutputStream buf = new BufferedOutputStream(sock.getOutputStream());        dout = new DataOutputStream(buf);
        // Sending id and challenge        // represents protocol version (in other words - message type)        dout.writeLong(PROTOCOL_VERSION);        dout.writeLong(self.getId());        String addr = self.getElectionAddress().getHostString() + ":" + self.getElectionAddress().getPort();        byte[] addr_bytes = addr.getBytes();        dout.writeInt(addr_bytes.length);        dout.write(addr_bytes);        dout.flush();
        din = new DataInputStream(                new BufferedInputStream(sock.getInputStream()));    } catch (IOException e) {        LOG.warn("Ignoring exception reading or writing challenge: ", e);        closeSocket(sock);        return false;    }
    // authenticate learner    QuorumPeer.QuorumServer qps = self.getVotingView().get(sid);    if (qps != null) {        // TODO - investigate why reconfig makes qps null.        authLearner.authenticate(sock, qps.hostname);    }
    // If lost the challenge, then drop the new connection    //保证集群中所有节点之间只有一个通道连接    if (sid > self.getId()) {        LOG.info("Have smaller server identifier, so dropping the " +                "connection: (" + sid + ", " + self.getId() + ")");        closeSocket(sock);        // Otherwise proceed with the connection    } else {        SendWorker sw = new SendWorker(sock, sid);        RecvWorker rw = new RecvWorker(sock, din, sid, sw);        sw.setRecv(rw);
        SendWorker vsw = senderWorkerMap.get(sid);
        if(vsw != null)            vsw.finish();
        senderWorkerMap.put(sid, sw);        queueSendMap.putIfAbsent(sid, new ArrayBlockingQueue<ByteBuffer>(                SEND_CAPACITY));
        sw.start();        rw.start();
        return true;
    }    return false;}

【Java】
public Vote lookForLeader() throws InterruptedException {//...    if (n.electionEpoch > logicalclock.get()) {        //当前选举周期小于选票周期，重置recvset选票池        //大于当前周期更新当前选票信息，再次发送投票        logicalclock.set(n.electionEpoch);        recvset.clear();        if(totalOrderPredicate(n.leader, n.zxid, n.peerEpoch,                getInitId(), getInitLastLoggedZxid(), getPeerEpoch())) {            updateProposal(n.leader, n.zxid, n.peerEpoch);        } else {            updateProposal(getInitId(),                    getInitLastLoggedZxid(),                    getPeerEpoch());        }        sendNotifications();    } else if (n.electionEpoch < logicalclock.get()) {        if(LOG.isDebugEnabled()){            LOG.debug("Notification election epoch is smaller than logicalclock. n.electionEpoch = 0x"                    + Long.toHexString(n.electionEpoch)                    + ", logicalclock=0x" + Long.toHexString(logicalclock.get()));        }        break;    } else if (totalOrderPredicate(n.leader, n.zxid, n.peerEpoch,            proposedLeader, proposedZxid, proposedEpoch)) {//相同选举周期        //接收的选票与当前选票PK成功后，替换当前选票        updateProposal(n.leader, n.zxid, n.peerEpoch);        sendNotifications();    }//...
}

【Java】//接收的选票与当前选票PKprotected boolean totalOrderPredicate(long newId, long newZxid, long newEpoch, long curId, long curZxid, long curEpoch) {        LOG.debug("id: " + newId + ", proposed id: " + curId + ", zxid: 0x" +                Long.toHexString(newZxid) + ", proposed zxid: 0x" + Long.toHexString(curZxid));        if(self.getQuorumVerifier().getWeight(newId) == 0){            return false;        }
        /*         * We return true if one of the following three cases hold:         * 1- New epoch is higher         * 2- New epoch is the same as current epoch, but new zxid is higher         * 3- New epoch is the same as current epoch, new zxid is the same         *  as current zxid, but server id is higher.         */        return ((newEpoch > curEpoch) ||                ((newEpoch == curEpoch) &&                ((newZxid > curZxid) || ((newZxid == curZxid) && (newId > curId)))));wId > curId)))));  }

【Java】public Vote lookForLeader() throws InterruptedException {//...   //存储节点对应的选票信息    // key:选票来源sid  value:选票推举的Leader sid    recvset.put(n.sid, new Vote(n.leader, n.zxid, n.electionEpoch, n.peerEpoch));
    //半数选举开始    if (termPredicate(recvset,            new Vote(proposedLeader, proposedZxid,                    logicalclock.get(), proposedEpoch))) {        // Verify if there is any change in the proposed leader        while((n = recvqueue.poll(finalizeWait,                TimeUnit.MILLISECONDS)) != null){            if(totalOrderPredicate(n.leader, n.zxid, n.peerEpoch,                    proposedLeader, proposedZxid, proposedEpoch)){                recvqueue.put(n);                break;            }        }        /*WorkerSender         * This predicate is true once we don't read any new         * relevant message from the reception queue         */        if (n == null) {            //已选举出leader 更新当前节点是否为leader             self.setPeerState((proposedLeader == self.getId()) ?                    ServerState.LEADING: learningState());
            Vote endVote = new Vote(proposedLeader,                    proposedZxid, proposedEpoch);            leaveInstance(endVote);            return endVote;        }    }//...}/**     * Termination predicate. Given a set of votes, determines if have     * sufficient to declare the end of the election round.     *     * @param votes     *            Set of votes     * @param vote     *            Identifier of the vote received last  PK后的选票     */private boolean termPredicate(HashMap<Long, Vote> votes, Vote vote) {    SyncedLearnerTracker voteSet = new SyncedLearnerTracker();    voteSet.addQuorumVerifier(self.getQuorumVerifier());    if (self.getLastSeenQuorumVerifier() != null            && self.getLastSeenQuorumVerifier().getVersion() > self                    .getQuorumVerifier().getVersion()) {        voteSet.addQuorumVerifier(self.getLastSeenQuorumVerifier());    }    /*     * First make the views consistent. Sometimes peers will have different     * zxids for a server depending on timing.     */    //votes 来源于recvset 存储各个节点推举出来的选票信息    for (Map.Entry<Long, Vote> entry : votes.entrySet()) {//选举出的sid和其它节点选择的sid相同存储到voteSet变量中。        if (vote.equals(entry.getValue())) {//保存推举出来的sid            voteSet.addAck(entry.getKey());        }    }    //判断选举出来的选票数量是否过半    return voteSet.hasAllQuorums();}//QuorumMaj#containsQuorumpublic boolean containsQuorum(Set<Long> ackSet) {    return (ackSet.size() > half);   }