| Kea 3.0.2-git

ha_service.cc
Go to the documentation of this file.
1// Copyright (C) 2018-2025 Internet Systems Consortium, Inc. ("ISC")
2//
3// This Source Code Form is subject to the terms of the Mozilla Public
4// License, v. 2.0. If a copy of the MPL was not distributed with this
5// file, You can obtain one at http://mozilla.org/MPL/2.0/.
6
7#include <config.h>
8
9#include <command_creator.h>
10#include <ha_log.h>
11#include <ha_service.h>
12#include <ha_service_states.h>
13#include <cc/command_interpreter.h>
14#include <cc/data.h>
15#include <config/cmd_response_creator.h>
16#include <config/timeouts.h>
17#include <dhcp/iface_mgr.h>
18#include <dhcpsrv/cfgmgr.h>
19#include <dhcpsrv/lease_mgr.h>
20#include <dhcpsrv/lease_mgr_factory.h>
21#include <exceptions/exceptions.h>
22#include <http/date_time.h>
23#include <http/response_json.h>
24#include <http/post_request_json.h>
25#include <util/boost_time_utils.h>
26#include <util/multi_threading_mgr.h>
27#include <util/stopwatch.h>
28#include <boost/pointer_cast.hpp>
29#include <boost/make_shared.hpp>
30#include <boost/weak_ptr.hpp>
31#include <functional>
32#include <sstream>
33
34 using namespace isc::asiolink;
35 using namespace isc::config;
36 using namespace isc::data;
37 using namespace isc::dhcp;
38 using namespace isc::hooks;
39 using namespace isc::http;
40 using namespace isc::log;
41 using namespace isc::util;
42 namespace ph = std::placeholders;
43
44 namespace {
45
47 class CommandUnsupportedError : public CtrlChannelError {
48 public:
49 CommandUnsupportedError(const char* file, size_t line, const char* what) :
50 CtrlChannelError(file, line, what) {}
51};
52
54 class ConflictError : public CtrlChannelError {
55 public:
56 ConflictError(const char* file, size_t line, const char* what) :
57 CtrlChannelError(file, line, what) {}
58};
59
60}
61
62 namespace isc {
63 namespace ha {
64
65 const int HAService::HA_HEARTBEAT_COMPLETE_EVT;
66 const int HAService::HA_LEASE_UPDATES_COMPLETE_EVT;
67 const int HAService::HA_SYNCING_FAILED_EVT;
68 const int HAService::HA_SYNCING_SUCCEEDED_EVT;
69 const int HAService::HA_MAINTENANCE_NOTIFY_EVT;
70 const int HAService::HA_MAINTENANCE_START_EVT;
71 const int HAService::HA_MAINTENANCE_CANCEL_EVT;
72 const int HAService::HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED;
73 const int HAService::HA_SYNCED_PARTNER_UNAVAILABLE_EVT;
74 const int HAService::HA_WAITING_TO_TERMINATED_ST_DELAY_MINUTES;
75
76 HAService::HAService(const unsigned int id, const IOServicePtr& io_service,
77 const NetworkStatePtr& network_state, const HAConfigPtr& config,
78 const HAServerType& server_type)
79 : id_(id), io_service_(io_service), network_state_(network_state), config_(config),
80 server_type_(server_type), client_(), listener_(), communication_state_(),
81 query_filter_(config), lease_sync_filter_(server_type, config), mutex_(),
82 pending_requests_(), lease_update_backlog_(config->getDelayedUpdatesLimit()),
83 sync_complete_notified_(false) {
84
85 if (server_type == HAServerType::DHCPv4) {
86 communication_state_.reset(new CommunicationState4(io_service_, config));
87
88 } else {
89 communication_state_.reset(new CommunicationState6(io_service_, config));
90 }
91
92 network_state_->enableService(getLocalOrigin());
93
94 startModel(HA_WAITING_ST);
95
96 // Create the client and(or) listener as appropriate.
97 if (!config_->getEnableMultiThreading()) {
98 // Not configured for multi-threading, start a client in ST mode.
99 client_.reset(new HttpClient(io_service_, false));
100 } else {
101 // Create an MT-mode client.
102 client_.reset(new HttpClient(io_service_, true,
103 config_->getHttpClientThreads(), true));
104
105 // If we're configured to use our own listener create and start it.
106 if (config_->getHttpDedicatedListener()) {
107 // Get the server address and port from this server's URL.
108 auto my_url = config_->getThisServerConfig()->getUrl();
109 IOAddress server_address(IOAddress::IPV4_ZERO_ADDRESS());
110 try {
111 // Since we do not currently support hostname resolution,
112 // we need to make sure we have an IP address here.
113 server_address = IOAddress(my_url.getStrippedHostname());
114 } catch (const std::exception& ex) {
115 isc_throw(Unexpected, "server Url:" << my_url.getStrippedHostname()
116 << " is not a valid IP address");
117 }
118
119 // Fetch how many threads the listener will use.
120 uint32_t listener_threads = config_->getHttpListenerThreads();
121
122 // Fetch the TLS context.
123 auto tls_context = config_->getThisServerConfig()->getTlsContext();
124
125 // Instantiate the listener.
126 listener_.reset(new CmdHttpListener(server_address, my_url.getPort(),
127 listener_threads, tls_context));
128 // Set the command filter when enabled.
129 if (config_->getRestrictCommands()) {
130 if (server_type == HAServerType::DHCPv4) {
131 CmdResponseCreator::command_accept_list_ =
132 CommandCreator::ha_commands4_;
133 } else {
134 CmdResponseCreator::command_accept_list_ =
135 CommandCreator::ha_commands6_;
136 }
137 }
138 }
139 }
140
141 LOG_INFO(ha_logger, HA_SERVICE_STARTED)
142 .arg(config_->getThisServerName())
143 .arg(HAConfig::HAModeToString(config->getHAMode()))
144 .arg(HAConfig::PeerConfig::roleToString(config->getThisServerConfig()->getRole()));
145}
146
147 HAService::~HAService() {
148 // Stop client and/or listener.
149 stopClientAndListener();
150
151 network_state_->enableService(getLocalOrigin());
152}
153
154std::string
155HAService::getCSCallbacksSetName() const {
156 std::ostringstream s;
157 s << "HA_MT_" << id_;
158 return (s.str());
159}
160
161 void
162 HAService::defineEvents() {
163 StateModel::defineEvents();
164
165 defineEvent(HA_HEARTBEAT_COMPLETE_EVT, "HA_HEARTBEAT_COMPLETE_EVT");
166 defineEvent(HA_LEASE_UPDATES_COMPLETE_EVT, "HA_LEASE_UPDATES_COMPLETE_EVT");
167 defineEvent(HA_SYNCING_FAILED_EVT, "HA_SYNCING_FAILED_EVT");
168 defineEvent(HA_SYNCING_SUCCEEDED_EVT, "HA_SYNCING_SUCCEEDED_EVT");
169 defineEvent(HA_MAINTENANCE_NOTIFY_EVT, "HA_MAINTENANCE_NOTIFY_EVT");
170 defineEvent(HA_MAINTENANCE_START_EVT, "HA_MAINTENANCE_START_EVT");
171 defineEvent(HA_MAINTENANCE_CANCEL_EVT, "HA_MAINTENANCE_CANCEL_EVT");
172 defineEvent(HA_SYNCED_PARTNER_UNAVAILABLE_EVT, "HA_SYNCED_PARTNER_UNAVAILABLE_EVT");
173}
174
175 void
176 HAService::verifyEvents() {
177 StateModel::verifyEvents();
178
179 getEvent(HA_HEARTBEAT_COMPLETE_EVT);
180 getEvent(HA_LEASE_UPDATES_COMPLETE_EVT);
181 getEvent(HA_SYNCING_FAILED_EVT);
182 getEvent(HA_SYNCING_SUCCEEDED_EVT);
183 getEvent(HA_MAINTENANCE_NOTIFY_EVT);
184 getEvent(HA_MAINTENANCE_START_EVT);
185 getEvent(HA_MAINTENANCE_CANCEL_EVT);
186 getEvent(HA_SYNCED_PARTNER_UNAVAILABLE_EVT);
187}
188
189 void
190 HAService::defineStates() {
191 StateModel::defineStates();
192
193 defineState(HA_BACKUP_ST, stateToString(HA_BACKUP_ST),
194 std::bind(&HAService::backupStateHandler, this),
195 config_->getStateMachineConfig()->getStateConfig(HA_BACKUP_ST)->getPausing());
196
197 defineState(HA_COMMUNICATION_RECOVERY_ST, stateToString(HA_COMMUNICATION_RECOVERY_ST),
198 std::bind(&HAService::communicationRecoveryHandler, this),
199 config_->getStateMachineConfig()->getStateConfig(HA_COMMUNICATION_RECOVERY_ST)->getPausing());
200
201 defineState(HA_HOT_STANDBY_ST, stateToString(HA_HOT_STANDBY_ST),
202 std::bind(&HAService::normalStateHandler, this),
203 config_->getStateMachineConfig()->getStateConfig(HA_HOT_STANDBY_ST)->getPausing());
204
205 defineState(HA_LOAD_BALANCING_ST, stateToString(HA_LOAD_BALANCING_ST),
206 std::bind(&HAService::normalStateHandler, this),
207 config_->getStateMachineConfig()->getStateConfig(HA_LOAD_BALANCING_ST)->getPausing());
208
209 defineState(HA_IN_MAINTENANCE_ST, stateToString(HA_IN_MAINTENANCE_ST),
210 std::bind(&HAService::inMaintenanceStateHandler, this),
211 config_->getStateMachineConfig()->getStateConfig(HA_IN_MAINTENANCE_ST)->getPausing());
212
213 defineState(HA_PARTNER_DOWN_ST, stateToString(HA_PARTNER_DOWN_ST),
214 std::bind(&HAService::partnerDownStateHandler, this),
215 config_->getStateMachineConfig()->getStateConfig(HA_PARTNER_DOWN_ST)->getPausing());
216
217 defineState(HA_PARTNER_IN_MAINTENANCE_ST, stateToString(HA_PARTNER_IN_MAINTENANCE_ST),
218 std::bind(&HAService::partnerInMaintenanceStateHandler, this),
219 config_->getStateMachineConfig()->getStateConfig(HA_PARTNER_IN_MAINTENANCE_ST)->getPausing());
220
221 defineState(HA_PASSIVE_BACKUP_ST, stateToString(HA_PASSIVE_BACKUP_ST),
222 std::bind(&HAService::passiveBackupStateHandler, this),
223 config_->getStateMachineConfig()->getStateConfig(HA_PASSIVE_BACKUP_ST)->getPausing());
224
225 defineState(HA_READY_ST, stateToString(HA_READY_ST),
226 std::bind(&HAService::readyStateHandler, this),
227 config_->getStateMachineConfig()->getStateConfig(HA_READY_ST)->getPausing());
228
229 defineState(HA_SYNCING_ST, stateToString(HA_SYNCING_ST),
230 std::bind(&HAService::syncingStateHandler, this),
231 config_->getStateMachineConfig()->getStateConfig(HA_SYNCING_ST)->getPausing());
232
233 defineState(HA_TERMINATED_ST, stateToString(HA_TERMINATED_ST),
234 std::bind(&HAService::terminatedStateHandler, this),
235 config_->getStateMachineConfig()->getStateConfig(HA_TERMINATED_ST)->getPausing());
236
237 defineState(HA_WAITING_ST, stateToString(HA_WAITING_ST),
238 std::bind(&HAService::waitingStateHandler, this),
239 config_->getStateMachineConfig()->getStateConfig(HA_WAITING_ST)->getPausing());
240}
241
242 void
243HAService::backupStateHandler() {
244 if (doOnEntry()) {
245 query_filter_.serveNoScopes();
246 adjustNetworkState();
247
248 // Log if the state machine is paused.
249 conditionalLogPausedState();
250 }
251
252 // There is nothing to do in that state. This server simply receives
253 // lease updates from the partners.
254 postNextEvent(NOP_EVT);
255}
256
257 void
258HAService::communicationRecoveryHandler() {
259 if (doOnEntry()) {
260 query_filter_.serveDefaultScopes();
261 adjustNetworkState();
262
263 // Log if the state machine is paused.
264 conditionalLogPausedState();
265 }
266
267 scheduleHeartbeat();
268
269 if (isMaintenanceCanceled() || isModelPaused()) {
270 postNextEvent(NOP_EVT);
271
272 // Check if the clock skew is still acceptable. If not, transition to
273 // the terminated state.
274 } else if (shouldTerminate()) {
275 verboseTransition(HA_TERMINATED_ST);
276
277 } else if (isPartnerStateInvalid()) {
278 verboseTransition(HA_WAITING_ST);
279
280 } else {
281
282 // Transitions based on the partner's state.
283 switch (communication_state_->getPartnerState()) {
284 case HA_IN_MAINTENANCE_ST:
285 verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST);
286 break;
287
288 case HA_PARTNER_DOWN_ST:
289 verboseTransition(HA_WAITING_ST);
290 break;
291
292 case HA_PARTNER_IN_MAINTENANCE_ST:
293 verboseTransition(HA_IN_MAINTENANCE_ST);
294 break;
295
296 case HA_TERMINATED_ST:
297 verboseTransition(HA_TERMINATED_ST);
298 break;
299
300 case HA_UNAVAILABLE_ST:
301 if (shouldPartnerDown()) {
302 verboseTransition(HA_PARTNER_DOWN_ST);
303
304 } else {
305 postNextEvent(NOP_EVT);
306 }
307 break;
308
309 case HA_WAITING_ST:
310 case HA_SYNCING_ST:
311 case HA_READY_ST:
312 // The partner seems to be waking up, perhaps after communication-recovery.
313 // If our backlog queue is overflown we need to synchronize our lease database.
314 // There is no need to send ha-reset to the partner because the partner is
315 // already synchronizing its lease database.
316 if (!communication_state_->isCommunicationInterrupted() &&
317 lease_update_backlog_.wasOverflown()) {
318 verboseTransition(HA_WAITING_ST);
319 } else {
320 // Backlog was not overflown, so there is no need to synchronize our
321 // lease database. Let's wait until our partner completes synchronization
322 // and transitions to the load-balancing state.
323 postNextEvent(NOP_EVT);
324 }
325 break;
326
327 default:
328 // If the communication is still interrupted, let's continue sitting
329 // in this state until it is resumed or until the transition to the
330 // partner-down state, depending on what happens first.
331 if (communication_state_->isCommunicationInterrupted()) {
332 postNextEvent(NOP_EVT);
333 break;
334 }
335
336 // The communication has been resumed. The partner server must be in a state
337 // in which it can receive outstanding lease updates we collected. The number of
338 // outstanding lease updates must not exceed the configured limit. Finally, the
339 // lease updates must be successfully sent. If that all works, we will transition
340 // to the normal operation.
341 if ((communication_state_->getPartnerState() == getNormalState()) ||
342 (communication_state_->getPartnerState() == HA_COMMUNICATION_RECOVERY_ST)) {
343 if (lease_update_backlog_.wasOverflown() || !sendLeaseUpdatesFromBacklog()) {
344 // If our lease backlog was overflown or we were unable to send lease
345 // updates to the partner we should notify the partner that it should
346 // synchronize the lease database. We do it by sending ha-reset command.
347 if (sendHAReset()) {
348 verboseTransition(HA_WAITING_ST);
349 }
350 break;
351 }
352 // The backlog was not overflown and we successfully sent our lease updates.
353 // We can now transition to the normal operation state. If the partner
354 // fails to send his outstanding lease updates to us it should send the
355 // ha-reset command to us.
356 verboseTransition(getNormalState());
357 break;
358 }
359
360 // The partner appears to be in unexpected state, we have exceeded the number
361 // of lease updates in a backlog or an attempt to send lease updates failed.
362 // In all these cases we follow plan B and transition to the waiting state.
363 // The server will then attempt to synchronize the entire lease database.
364 verboseTransition(HA_WAITING_ST);
365 }
366 }
367
368 // When exiting this state we must ensure that lease updates backlog is cleared.
369 if (doOnExit()) {
370 lease_update_backlog_.clear();
371 }
372}
373
374 void
375HAService::normalStateHandler() {
376 // If we are transitioning from another state, we have to define new
377 // serving scopes appropriate for the new state. We don't do it if
378 // we remain in this state.
379 if (doOnEntry()) {
380 query_filter_.serveDefaultScopes();
381 adjustNetworkState();
382
383 // Log if the state machine is paused.
384 conditionalLogPausedState();
385 }
386
387 scheduleHeartbeat();
388
389 if (isMaintenanceCanceled() || isModelPaused()) {
390 postNextEvent(NOP_EVT);
391 return;
392 }
393
394 // Check if the clock skew is still acceptable. If not, transition to
395 // the terminated state.
396 if (shouldTerminate()) {
397 verboseTransition(HA_TERMINATED_ST);
398 return;
399 }
400
401 // Check if the partner state is valid per current configuration. If it is
402 // in an invalid state let's transition to the waiting state and stay there
403 // until the configuration is corrected.
404 if (isPartnerStateInvalid()) {
405 verboseTransition(HA_WAITING_ST);
406 return;
407 }
408
409 switch (communication_state_->getPartnerState()) {
410 case HA_IN_MAINTENANCE_ST:
411 verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST);
412 break;
413
414 case HA_PARTNER_DOWN_ST:
415 verboseTransition(HA_WAITING_ST);
416 break;
417
418 case HA_PARTNER_IN_MAINTENANCE_ST:
419 verboseTransition(HA_IN_MAINTENANCE_ST);
420 break;
421
422 case HA_TERMINATED_ST:
423 verboseTransition(HA_TERMINATED_ST);
424 break;
425
426 case HA_UNAVAILABLE_ST:
427 if (shouldPartnerDown()) {
428 verboseTransition(HA_PARTNER_DOWN_ST);
429
430 } else if (config_->amAllowingCommRecovery()) {
431 verboseTransition(HA_COMMUNICATION_RECOVERY_ST);
432
433 } else {
434 postNextEvent(NOP_EVT);
435 }
436 break;
437
438 default:
439 postNextEvent(NOP_EVT);
440 }
441
442 if (doOnExit()) {
443 // Do nothing here but doOnExit() call clears the "on exit" flag
444 // when transitioning to the communication-recovery state. In that
445 // state we need this flag to be cleared.
446 }
447}
448
449 void
450HAService::inMaintenanceStateHandler() {
451 // If we are transitioning from another state, we have to define new
452 // serving scopes appropriate for the new state. We don't do it if
453 // we remain in this state.
454 if (doOnEntry()) {
455 // In this state the server remains silent and waits for being
456 // shutdown.
457 query_filter_.serveNoScopes();
458 adjustNetworkState();
459
460 // Log if the state machine is paused.
461 conditionalLogPausedState();
462
463 LOG_INFO(ha_logger, HA_MAINTENANCE_SHUTDOWN_SAFE)
464 .arg(config_->getThisServerName());
465 }
466
467 scheduleHeartbeat();
468
469 // We don't transition out of this state unless explicitly mandated
470 // by the administrator via a dedicated command which cancels
471 // the maintenance.
472 postNextEvent(NOP_EVT);
473}
474
475 void
476HAService::partnerDownStateHandler() {
477 // If we are transitioning from another state, we have to define new
478 // serving scopes appropriate for the new state. We don't do it if
479 // we remain in this state.
480 if (doOnEntry()) {
481
482 bool maintenance = (getLastEvent() == HA_MAINTENANCE_START_EVT);
483
484 // It may be administratively disabled to handle partner's scope
485 // in case of failure. If this is the case we'll just handle our
486 // default scope (or no scope at all). The user will need to
487 // manually enable this server to handle partner's scope.
488 // If we're in the maintenance mode we serve all scopes because
489 // it is not a failover situation.
490 if (maintenance || config_->getThisServerConfig()->isAutoFailover()) {
491 query_filter_.serveFailoverScopes();
492 } else {
493 query_filter_.serveDefaultScopes();
494 }
495 adjustNetworkState();
496 communication_state_->clearRejectedLeaseUpdates();
497
498 // Log if the state machine is paused.
499 conditionalLogPausedState();
500
501 if (maintenance) {
502 // If we ended up in the partner-down state as a result of
503 // receiving the ha-maintenance-start command let's log it.
504 LOG_INFO(ha_logger, HA_MAINTENANCE_STARTED_IN_PARTNER_DOWN)
505 .arg(config_->getThisServerName());
506 }
507
508 } else if (getLastEvent() == HA_SYNCED_PARTNER_UNAVAILABLE_EVT) {
509 // Partner sent the ha-sync-complete-notify command to indicate that
510 // it has successfully synchronized its lease database but this server
511 // was unable to send heartbeat to this server. Enable the DHCP service
512 // and continue serving the clients in the partner-down state until the
513 // communication with the partner is fixed.
514 adjustNetworkState();
515 }
516
517 scheduleHeartbeat();
518
519 if (isMaintenanceCanceled() || isModelPaused()) {
520 postNextEvent(NOP_EVT);
521 return;
522 }
523
524 // Check if the clock skew is still acceptable. If not, transition to
525 // the terminated state.
526 if (shouldTerminate()) {
527 verboseTransition(HA_TERMINATED_ST);
528 return;
529 }
530
531 // Check if the partner state is valid per current configuration. If it is
532 // in an invalid state let's transition to the waiting state and stay there
533 // until the configuration is corrected.
534 if (isPartnerStateInvalid()) {
535 verboseTransition(HA_WAITING_ST);
536 return;
537 }
538
539 switch (communication_state_->getPartnerState()) {
540 case HA_COMMUNICATION_RECOVERY_ST:
541 case HA_PARTNER_DOWN_ST:
542 case HA_PARTNER_IN_MAINTENANCE_ST:
543 verboseTransition(HA_WAITING_ST);
544 break;
545
546 case HA_READY_ST:
547 // If partner allocated new leases for which it didn't send lease updates
548 // to us we should synchronize our database.
549 if (communication_state_->hasPartnerNewUnsentUpdates()) {
550 verboseTransition(HA_WAITING_ST);
551 } else {
552 // We did not miss any lease updates. There is no need to synchronize
553 // the database.
554 verboseTransition(getNormalState());
555 }
556 break;
557
558 case HA_TERMINATED_ST:
559 verboseTransition(HA_TERMINATED_ST);
560 break;
561
562 default:
563 postNextEvent(NOP_EVT);
564 }
565}
566
567 void
568HAService::partnerInMaintenanceStateHandler() {
569 // If we are transitioning from another state, we have to define new
570 // serving scopes appropriate for the new state. We don't do it if
571 // we remain in this state.
572 if (doOnEntry()) {
573 query_filter_.serveFailoverScopes();
574
575 adjustNetworkState();
576
577 // Log if the state machine is paused.
578 conditionalLogPausedState();
579
580 LOG_INFO(ha_logger, HA_MAINTENANCE_STARTED)
581 .arg(config_->getThisServerName());
582 }
583
584 scheduleHeartbeat();
585
586 if (isModelPaused()) {
587 postNextEvent(NOP_EVT);
588 return;
589 }
590
591 // Check if the clock skew is still acceptable. If not, transition to
592 // the terminated state.
593 if (shouldTerminate()) {
594 verboseTransition(HA_TERMINATED_ST);
595 return;
596 }
597
598 switch (communication_state_->getPartnerState()) {
599 case HA_UNAVAILABLE_ST:
600 verboseTransition(HA_PARTNER_DOWN_ST);
601 break;
602 default:
603 postNextEvent(NOP_EVT);
604 }
605}
606
607 void
608HAService::passiveBackupStateHandler() {
609 // If we are transitioning from another state, we have to define new
610 // serving scopes appropriate for the new state. We don't do it if
611 // we remain in this state.
612 if (doOnEntry()) {
613 query_filter_.serveDefaultScopes();
614 adjustNetworkState();
615
616 // In the passive-backup state we don't send heartbeat.
617 communication_state_->stopHeartbeat();
618
619 // Log if the state machine is paused.
620 conditionalLogPausedState();
621 }
622 postNextEvent(NOP_EVT);
623}
624
625 void
626HAService::readyStateHandler() {
627 // If we are transitioning from another state, we have to define new
628 // serving scopes appropriate for the new state. We don't do it if
629 // we remain in this state.
630 if (doOnEntry()) {
631 query_filter_.serveNoScopes();
632 adjustNetworkState();
633 communication_state_->clearRejectedLeaseUpdates();
634
635 // Log if the state machine is paused.
636 conditionalLogPausedState();
637 }
638
639 scheduleHeartbeat();
640
641 if (isMaintenanceCanceled() || isModelPaused()) {
642 postNextEvent(NOP_EVT);
643 return;
644 }
645
646 // Check if the clock skew is still acceptable. If not, transition to
647 // the terminated state.
648 if (shouldTerminate()) {
649 verboseTransition(HA_TERMINATED_ST);
650 return;
651 }
652
653 // Check if the partner state is valid per current configuration. If it is
654 // in an invalid state let's transition to the waiting state and stay there
655 // until the configuration is corrected.
656 if (isPartnerStateInvalid()) {
657 verboseTransition(HA_WAITING_ST);
658 return;
659 }
660
661 switch (communication_state_->getPartnerState()) {
662 case HA_HOT_STANDBY_ST:
663 case HA_LOAD_BALANCING_ST:
664 case HA_COMMUNICATION_RECOVERY_ST:
665 verboseTransition(getNormalState());
666 break;
667
668 case HA_IN_MAINTENANCE_ST:
669 verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST);
670 break;
671
672 case HA_PARTNER_IN_MAINTENANCE_ST:
673 verboseTransition(HA_IN_MAINTENANCE_ST);
674 break;
675
676 case HA_READY_ST:
677 // If both servers are ready, the primary server "wins" and is
678 // transitioned first.
679 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::PRIMARY) {
680 verboseTransition((config_->getHAMode() == HAConfig::LOAD_BALANCING ?
681 HA_LOAD_BALANCING_ST : HA_HOT_STANDBY_ST));
682 } else {
683 postNextEvent(NOP_EVT);
684 }
685 break;
686
687 case HA_TERMINATED_ST:
688 verboseTransition(HA_TERMINATED_ST);
689 break;
690
691 case HA_UNAVAILABLE_ST:
692 if (shouldPartnerDown()) {
693 verboseTransition(HA_PARTNER_DOWN_ST);
694
695 } else {
696 postNextEvent(NOP_EVT);
697 }
698 break;
699
700 default:
701 postNextEvent(NOP_EVT);
702 }
703}
704
705 void
706HAService::syncingStateHandler() {
707 // If we are transitioning from another state, we have to define new
708 // serving scopes appropriate for the new state. We don't do it if
709 // we remain in this state.
710 if (doOnEntry()) {
711 query_filter_.serveNoScopes();
712 adjustNetworkState();
713 communication_state_->clearRejectedLeaseUpdates();
714
715 // Log if the state machine is paused.
716 conditionalLogPausedState();
717 }
718
719 if (isMaintenanceCanceled() || isModelPaused()) {
720 postNextEvent(NOP_EVT);
721 return;
722 }
723
724 // Check if the clock skew is still acceptable. If not, transition to
725 // the terminated state.
726 if (shouldTerminate()) {
727 verboseTransition(HA_TERMINATED_ST);
728 return;
729 }
730
731 // Check if the partner state is valid per current configuration. If it is
732 // in an invalid state let's transition to the waiting state and stay there
733 // until the configuration is corrected.
734 if (isPartnerStateInvalid()) {
735 verboseTransition(HA_WAITING_ST);
736 return;
737 }
738
739 // We don't want to perform synchronous attempt to synchronize with
740 // a partner until we know that the partner is responding. Therefore,
741 // we wait for the heartbeat to complete successfully before we
742 // initiate the synchronization.
743 switch (communication_state_->getPartnerState()) {
744 case HA_TERMINATED_ST:
745 verboseTransition(HA_TERMINATED_ST);
746 return;
747
748 case HA_UNAVAILABLE_ST:
749 // If the partner appears to be offline, let's transition to the partner
750 // down state. Otherwise, we'd be stuck trying to synchronize with a
751 // dead partner.
752 if (shouldPartnerDown()) {
753 verboseTransition(HA_PARTNER_DOWN_ST);
754
755 } else {
756 postNextEvent(NOP_EVT);
757 }
758 break;
759
760 default:
761 // We don't want the heartbeat to interfere with the synchronization,
762 // so let's temporarily stop it.
763 communication_state_->stopHeartbeat();
764
765 // Timeout is configured in milliseconds. Need to convert to seconds.
766 unsigned int dhcp_disable_timeout =
767 static_cast<unsigned int>(config_->getSyncTimeout() / 1000);
768 if (dhcp_disable_timeout == 0) {
769 ++dhcp_disable_timeout;
770 }
771
772 // Perform synchronous leases update.
773 std::string status_message;
774 int sync_status = synchronize(status_message,
775 config_->getFailoverPeerConfig(),
776 dhcp_disable_timeout);
777
778 // If the leases synchronization was successful, let's transition
779 // to the ready state.
780 if (sync_status == CONTROL_RESULT_SUCCESS) {
781 verboseTransition(HA_READY_ST);
782
783 } else {
784 // If the synchronization was unsuccessful we're back to the
785 // situation that the partner is unavailable and therefore
786 // we stay in the syncing state.
787 postNextEvent(NOP_EVT);
788 }
789 }
790
791 // Make sure that the heartbeat is re-enabled.
792 scheduleHeartbeat();
793}
794
795 void
796HAService::terminatedStateHandler() {
797 // If we are transitioning from another state, we have to define new
798 // serving scopes appropriate for the new state. We don't do it if
799 // we remain in this state.
800 if (doOnEntry()) {
801 query_filter_.serveDefaultScopes();
802 adjustNetworkState();
803 communication_state_->clearRejectedLeaseUpdates();
804
805 // In the terminated state we don't send heartbeat.
806 communication_state_->stopHeartbeat();
807
808 // Log if the state machine is paused.
809 conditionalLogPausedState();
810
811 LOG_ERROR(ha_logger, HA_TERMINATED)
812 .arg(config_->getThisServerName());
813 }
814
815 postNextEvent(NOP_EVT);
816}
817
818 void
819HAService::waitingStateHandler() {
820 // If we are transitioning from another state, we have to define new
821 // serving scopes appropriate for the new state. We don't do it if
822 // we remain in this state.
823 if (doOnEntry()) {
824 query_filter_.serveNoScopes();
825 adjustNetworkState();
826 communication_state_->clearRejectedLeaseUpdates();
827
828 // Log if the state machine is paused.
829 conditionalLogPausedState();
830 }
831
832 // Only schedule the heartbeat for non-backup servers.
833 if ((config_->getHAMode() != HAConfig::PASSIVE_BACKUP) &&
834 (config_->getThisServerConfig()->getRole() != HAConfig::PeerConfig::BACKUP)) {
835 scheduleHeartbeat();
836 }
837
838 if (isMaintenanceCanceled() || isModelPaused()) {
839 postNextEvent(NOP_EVT);
840 return;
841 }
842
843 // Backup server must remain in its own state.
844 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
845 verboseTransition(HA_BACKUP_ST);
846 return;
847 }
848
849 // We're not a backup server, so we're either primary or secondary. If this is
850 // a passive-backup mode of operation, we're primary and we should transition
851 // to the passive-backup state.
852 if (config_->getHAMode() == HAConfig::PASSIVE_BACKUP) {
853 verboseTransition(HA_PASSIVE_BACKUP_ST);
854 return;
855 }
856
857 // Check if the clock skew is still acceptable. If not, transition to
858 // the terminated state.
859 if (shouldTerminate()) {
860 verboseTransition(HA_TERMINATED_ST);
861 return;
862 }
863
864 // Check if the partner state is valid per current configuration. If it is
865 // in an invalid state let's sit in the waiting state until the configuration
866 // is corrected.
867 if (isPartnerStateInvalid()) {
868 postNextEvent(NOP_EVT);
869 return;
870 }
871
872 switch (communication_state_->getPartnerState()) {
873 case HA_COMMUNICATION_RECOVERY_ST:
874 case HA_HOT_STANDBY_ST:
875 case HA_LOAD_BALANCING_ST:
876 case HA_IN_MAINTENANCE_ST:
877 case HA_PARTNER_DOWN_ST:
878 case HA_PARTNER_IN_MAINTENANCE_ST:
879 case HA_READY_ST:
880 // If we're configured to not synchronize lease database, proceed directly
881 // to the "ready" state.
882 verboseTransition(config_->amSyncingLeases() ? HA_SYNCING_ST : HA_READY_ST);
883 break;
884
885 case HA_SYNCING_ST:
886 postNextEvent(NOP_EVT);
887 break;
888
889 case HA_TERMINATED_ST: {
890 auto partner_in_terminated = communication_state_->getDurationSincePartnerStateTime();
891 if (!partner_in_terminated.is_not_a_date_time() &&
892 (partner_in_terminated.total_seconds()) / 60 >= HA_WAITING_TO_TERMINATED_ST_DELAY_MINUTES) {
893 LOG_WARN(ha_logger, HA_TERMINATED_PARTNER_DID_NOT_RESTART)
894 .arg(config_->getThisServerName())
895 .arg(HA_WAITING_TO_TERMINATED_ST_DELAY_MINUTES);
896 verboseTransition(HA_TERMINATED_ST);
897 break;
898 }
899
900 // We have checked above whether the clock skew is exceeding the threshold
901 // and we should terminate. If we're here, it means that the clock skew
902 // is acceptable. The partner may be still in the terminated state because
903 // it hasn't been restarted yet. Probably, this server is the first one
904 // being restarted after syncing the clocks. Let's just sit in the waiting
905 // state until the partner gets restarted.
906 LOG_INFO(ha_logger, HA_TERMINATED_RESTART_PARTNER)
907 .arg(config_->getThisServerName());
908 postNextEvent(NOP_EVT);
909 break;
910 }
911 case HA_WAITING_ST:
912 // If both servers are waiting, the primary server 'wins' and is
913 // transitioned to the next state first.
914 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::PRIMARY) {
915 // If we're configured to not synchronize lease database, proceed directly
916 // to the "ready" state.
917 verboseTransition(config_->amSyncingLeases() ? HA_SYNCING_ST : HA_READY_ST);
918
919 } else {
920 postNextEvent(NOP_EVT);
921 }
922 break;
923
924 case HA_UNAVAILABLE_ST:
925 if (shouldPartnerDown()) {
926 verboseTransition(HA_PARTNER_DOWN_ST);
927
928 } else {
929 postNextEvent(NOP_EVT);
930 }
931 break;
932
933 default:
934 postNextEvent(NOP_EVT);
935 }
936}
937
938 void
939 HAService::verboseTransition(const unsigned state) {
940 // Get current and new state name.
941 std::string current_state_name = getStateLabel(getCurrState());
942 std::string new_state_name = getStateLabel(state);
943
944 // Turn them to upper case so as they are better visible in the logs.
945 boost::to_upper(current_state_name);
946 boost::to_upper(new_state_name);
947
948 if (config_->getHAMode() != HAConfig::PASSIVE_BACKUP) {
949 // If this is load-balancing or hot-standby mode we also want to log
950 // partner's state.
951 auto partner_state = communication_state_->getPartnerState();
952 std::string partner_state_name = getStateLabel(partner_state);
953 boost::to_upper(partner_state_name);
954
955 // Log the transition.
956 LOG_INFO(ha_logger, HA_STATE_TRANSITION)
957 .arg(config_->getThisServerName())
958 .arg(current_state_name)
959 .arg(new_state_name)
960 .arg(partner_state_name);
961
962 } else {
963 // In the passive-backup mode we don't know the partner's state.
964 LOG_INFO(ha_logger, HA_STATE_TRANSITION_PASSIVE_BACKUP)
965 .arg(config_->getThisServerName())
966 .arg(current_state_name)
967 .arg(new_state_name);
968 }
969
970 // If we're transitioning directly from the "waiting" to "ready"
971 // state it indicates that the database synchronization is
972 // administratively disabled. Let's remind the user about this
973 // configuration setting.
974 if ((state == HA_READY_ST) && (getCurrState() == HA_WAITING_ST)) {
975 LOG_INFO(ha_logger, HA_CONFIG_LEASE_SYNCING_DISABLED_REMINDER)
976 .arg(config_->getThisServerName());
977 }
978
979 // Do the actual transition.
980 transition(state, getNextEvent());
981
982 // Inform the administrator whether or not lease updates are generated.
983 // Updates are never generated by a backup server so it doesn't make
984 // sense to log anything for the backup server.
985 if ((config_->getHAMode() != HAConfig::PASSIVE_BACKUP) &&
986 (config_->getThisServerConfig()->getRole() != HAConfig::PeerConfig::BACKUP)) {
987 if (shouldSendLeaseUpdates(config_->getFailoverPeerConfig())) {
988 LOG_INFO(ha_logger, HA_LEASE_UPDATES_ENABLED)
989 .arg(config_->getThisServerName())
990 .arg(new_state_name);
991
992 } else if (!config_->amSendingLeaseUpdates()) {
993 // Lease updates are administratively disabled.
994 LOG_INFO(ha_logger, HA_CONFIG_LEASE_UPDATES_DISABLED_REMINDER)
995 .arg(config_->getThisServerName())
996 .arg(new_state_name);
997
998 } else {
999 // Lease updates are not administratively disabled, but they
1000 // are not issued because this is the backup server or because
1001 // in this state the server should not generate lease updates.
1002 LOG_INFO(ha_logger, HA_LEASE_UPDATES_DISABLED)
1003 .arg(config_->getThisServerName())
1004 .arg(new_state_name);
1005 }
1006 }
1007}
1008
1009 int
1010 HAService::getNormalState() const {
1011 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
1012 return (HA_BACKUP_ST);
1013 }
1014
1015 switch (config_->getHAMode()) {
1016 case HAConfig::LOAD_BALANCING:
1017 return (HA_LOAD_BALANCING_ST);
1018 case HAConfig::HOT_STANDBY:
1019 return (HA_HOT_STANDBY_ST);
1020 default:
1021 return (HA_PASSIVE_BACKUP_ST);
1022 }
1023}
1024
1025 bool
1026 HAService::unpause() {
1027 if (isModelPaused()) {
1028 LOG_INFO(ha_logger, HA_STATE_MACHINE_CONTINUED)
1029 .arg(config_->getThisServerName());
1030 unpauseModel();
1031 return (true);
1032 }
1033 return (false);
1034}
1035
1036 void
1037 HAService::conditionalLogPausedState() const {
1038 // Inform the administrator if the state machine is paused.
1039 if (isModelPaused()) {
1040 std::string state_name = stateToString(getCurrState());
1041 boost::to_upper(state_name);
1042 LOG_INFO(ha_logger, HA_STATE_MACHINE_PAUSED)
1043 .arg(config_->getThisServerName())
1044 .arg(state_name);
1045 }
1046}
1047
1048 void
1049 HAService::serveDefaultScopes() {
1050 query_filter_.serveDefaultScopes();
1051}
1052
1053 void
1054 HAService::serveFailoverScopes() {
1055 query_filter_.serveFailoverScopes();
1056}
1057
1058 bool
1059 HAService::inScope(dhcp::Pkt4Ptr& query4) {
1060 return (inScopeInternal(query4));
1061}
1062
1063 bool
1064 HAService::inScope(dhcp::Pkt6Ptr& query6) {
1065 return (inScopeInternal(query6));
1066}
1067
1068 template<typename QueryPtrType>
1069 bool
1070HAService::inScopeInternal(QueryPtrType& query) {
1071 // Check if the query is in scope (should be processed by this server).
1072 std::string scope_class;
1073 const bool in_scope = query_filter_.inScope(query, scope_class);
1074 // Whether or not the query is going to be processed by this server,
1075 // we associate the query with the appropriate class.
1076 query->addClass(dhcp::ClientClass(scope_class));
1077 // The following is the part of the server failure detection algorithm.
1078 // If the query should be processed by the partner we need to check if
1079 // the partner responds. If the number of unanswered queries exceeds a
1080 // configured threshold, we will consider the partner to be offline.
1081 if (!in_scope && communication_state_->isCommunicationInterrupted()) {
1082 communication_state_->analyzeMessage(query);
1083 }
1084 // Indicate if the query is in scope.
1085 return (in_scope);
1086}
1087
1088 bool
1089 HAService::shouldReclaim(const dhcp::Lease4Ptr& lease4) const {
1090 return (shouldReclaimInternal(lease4));
1091}
1092
1093 bool
1094 HAService::shouldReclaim(const dhcp::Lease6Ptr& lease6) const {
1095 return (shouldReclaimInternal(lease6));
1096}
1097
1098 template<typename LeasePtrType>
1099 bool
1100HAService::shouldReclaimInternal(const LeasePtrType& lease) const {
1101 return (getCurrState() != HA_TERMINATED_ST || query_filter_.inScope(lease));
1102}
1103
1104 void
1105 HAService::adjustNetworkState() {
1106 std::string current_state_name = getStateLabel(getCurrState());
1107 boost::to_upper(current_state_name);
1108
1109 // DHCP service should be enabled in the following states.
1110 const bool should_enable = ((getCurrState() == HA_COMMUNICATION_RECOVERY_ST) ||
1111 (getCurrState() == HA_LOAD_BALANCING_ST) ||
1112 (getCurrState() == HA_HOT_STANDBY_ST) ||
1113 (getCurrState() == HA_PARTNER_DOWN_ST) ||
1114 (getCurrState() == HA_PARTNER_IN_MAINTENANCE_ST) ||
1115 (getCurrState() == HA_PASSIVE_BACKUP_ST) ||
1116 (getCurrState() == HA_TERMINATED_ST));
1117
1118 if (!should_enable && network_state_->isServiceEnabled()) {
1119 current_state_name = getStateLabel(getCurrState());
1120 boost::to_upper(current_state_name);
1121 LOG_INFO(ha_logger, HA_LOCAL_DHCP_DISABLE)
1122 .arg(config_->getThisServerName())
1123 .arg(current_state_name);
1124 network_state_->disableService(getLocalOrigin());
1125
1126 } else if (should_enable && !network_state_->isServiceEnabled()) {
1127 current_state_name = getStateLabel(getCurrState());
1128 boost::to_upper(current_state_name);
1129 LOG_INFO(ha_logger, HA_LOCAL_DHCP_ENABLE)
1130 .arg(config_->getThisServerName())
1131 .arg(current_state_name);
1132 network_state_->enableService(getLocalOrigin());
1133 }
1134}
1135
1136 bool
1137 HAService::shouldPartnerDown() const {
1138 // Checking whether the communication with the partner is OK is the
1139 // first step towards verifying if the server is up.
1140 if (communication_state_->isCommunicationInterrupted()) {
1141 // If the communication is interrupted, we also have to check
1142 // whether the partner answers DHCP requests. The only cases
1143 // when we don't (can't) do it are: the hot standby configuration
1144 // in which this server is a primary and when the DHCP service is
1145 // disabled so we can't analyze incoming traffic. Note that the
1146 // primary server can't check delayed responses to the partner
1147 // because the partner doesn't respond to any queries in this
1148 // configuration.
1149 if (network_state_->isServiceEnabled() &&
1150 ((config_->getHAMode() == HAConfig::LOAD_BALANCING) ||
1151 (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::STANDBY))) {
1152 return (communication_state_->failureDetected());
1153 }
1154
1155 // Hot standby / primary case.
1156 return (true);
1157 }
1158
1159 // Shouldn't transition to the partner down state.
1160 return (false);
1161}
1162
1163 bool
1164 HAService::shouldTerminate() const {
1165 // Check if skew is fatally large.
1166 bool should_terminate = communication_state_->clockSkewShouldTerminate();
1167
1168 // If not issue a warning if it's getting large.
1169 if (!should_terminate) {
1170 communication_state_->clockSkewShouldWarn();
1171 // Check if we should terminate because the number of rejected leases
1172 // has been exceeded.
1173 should_terminate = communication_state_->rejectedLeaseUpdatesShouldTerminate();
1174 }
1175
1176 return (should_terminate);
1177}
1178
1179 bool
1180 HAService::isMaintenanceCanceled() const {
1181 return (getLastEvent() == HA_MAINTENANCE_CANCEL_EVT);
1182}
1183
1184 bool
1185 HAService::isPartnerStateInvalid() const {
1186 switch (communication_state_->getPartnerState()) {
1187 case HA_COMMUNICATION_RECOVERY_ST:
1188 if (config_->getHAMode() != HAConfig::LOAD_BALANCING) {
1189 LOG_WARN(ha_logger, HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY)
1190 .arg(config_->getThisServerName());
1191 return (true);
1192 }
1193 break;
1194
1195 case HA_HOT_STANDBY_ST:
1196 if (config_->getHAMode() != HAConfig::HOT_STANDBY) {
1197 LOG_WARN(ha_logger, HA_INVALID_PARTNER_STATE_HOT_STANDBY)
1198 .arg(config_->getThisServerName());
1199 return (true);
1200 }
1201 break;
1202
1203 case HA_LOAD_BALANCING_ST:
1204 if (config_->getHAMode() != HAConfig::LOAD_BALANCING) {
1205 LOG_WARN(ha_logger, HA_INVALID_PARTNER_STATE_LOAD_BALANCING)
1206 .arg(config_->getThisServerName());
1207 return (true);
1208 }
1209 break;
1210
1211 default:
1212 ;
1213 }
1214 return (false);
1215}
1216
1217 size_t
1218 HAService::asyncSendLeaseUpdates(const dhcp::Pkt4Ptr& query,
1219 const dhcp::Lease4CollectionPtr& leases,
1220 const dhcp::Lease4CollectionPtr& deleted_leases,
1221 const hooks::ParkingLotHandlePtr& parking_lot) {
1222
1223 // Get configurations of the peers. Exclude this instance.
1224 HAConfig::PeerConfigMap peers_configs = config_->getOtherServersConfig();
1225
1226 size_t sent_num = 0;
1227
1228 // Schedule sending lease updates to each peer.
1229 for (auto const& p : peers_configs) {
1230 HAConfig::PeerConfigPtr conf = p.second;
1231
1232 // Check if the lease updates should be queued. This is the case when the
1233 // server is in the communication-recovery state. Queued lease updates may
1234 // be sent when the communication is re-established.
1235 if (shouldQueueLeaseUpdates(conf)) {
1236 // Lease updates for deleted leases.
1237 for (auto const& l : *deleted_leases) {
1238 // If a released lease is preserved in the database send the lease
1239 // update to the partner. Otherwise, delete the lease.
1240 if (l->state_ == Lease4::STATE_RELEASED) {
1241 lease_update_backlog_.push(LeaseUpdateBacklog::ADD, l);
1242 } else {
1243 lease_update_backlog_.push(LeaseUpdateBacklog::DELETE, l);
1244 }
1245 }
1246
1247 // Lease updates for new allocations and updated leases.
1248 for (auto const& l : *leases) {
1249 lease_update_backlog_.push(LeaseUpdateBacklog::ADD, l);
1250 }
1251
1252 continue;
1253 }
1254
1255 // Check if the lease update should be sent to the server. If we're in
1256 // the partner-down state we don't send lease updates to the partner.
1257 if (!shouldSendLeaseUpdates(conf)) {
1258 // If we decide to not send the lease updates to an active partner, we
1259 // should make a record of it in the communication state. The partner
1260 // can check if there were any unsent lease updates when he determines
1261 // whether it should synchronize its database or not when it recovers
1262 // from the partner-down state.
1263 if (conf->getRole() != HAConfig::PeerConfig::BACKUP) {
1264 communication_state_->increaseUnsentUpdateCount();
1265 }
1266 continue;
1267 }
1268
1269 // Lease updates for deleted leases.
1270 for (auto const& l : *deleted_leases) {
1271 // If a released lease is preserved in the database send the lease
1272 // update to the partner. Otherwise, delete the lease.
1273 if (l->state_ == Lease4::STATE_RELEASED) {
1274 asyncSendLeaseUpdate(query, conf, CommandCreator::createLease4Update(*l),
1275 parking_lot);
1276 } else {
1277 asyncSendLeaseUpdate(query, conf, CommandCreator::createLease4Delete(*l),
1278 parking_lot);
1279 }
1280 }
1281
1282 // Lease updates for new allocations and updated leases.
1283 for (auto const& l : *leases) {
1284 asyncSendLeaseUpdate(query, conf, CommandCreator::createLease4Update(*l),
1285 parking_lot);
1286 }
1287
1288 // If we're contacting a backup server from which we don't expect a
1289 // response prior to responding to the DHCP client we don't count
1290 // it.
1291 if ((config_->amWaitingBackupAck() || (conf->getRole() != HAConfig::PeerConfig::BACKUP))) {
1292 ++sent_num;
1293 }
1294 }
1295
1296 return (sent_num);
1297}
1298
1299 size_t
1300 HAService::asyncSendSingleLeaseUpdate(const dhcp::Pkt4Ptr& query,
1301 const dhcp::Lease4Ptr& lease,
1302 const hooks::ParkingLotHandlePtr& parking_lot) {
1303 Lease4CollectionPtr leases(new Lease4Collection());
1304 leases->push_back(lease);
1305 Lease4CollectionPtr deleted_leases(new Lease4Collection());
1306
1307 return (asyncSendLeaseUpdates(query, leases, deleted_leases, parking_lot));
1308}
1309
1310 size_t
1311 HAService::asyncSendLeaseUpdates(const dhcp::Pkt6Ptr& query,
1312 const dhcp::Lease6CollectionPtr& leases,
1313 const dhcp::Lease6CollectionPtr& deleted_leases,
1314 const hooks::ParkingLotHandlePtr& parking_lot) {
1315
1316 // Get configurations of the peers. Exclude this instance.
1317 HAConfig::PeerConfigMap peers_configs = config_->getOtherServersConfig();
1318
1319 size_t sent_num = 0;
1320
1321 // Schedule sending lease updates to each peer.
1322 for (auto const& p : peers_configs) {
1323 HAConfig::PeerConfigPtr conf = p.second;
1324
1325 // Check if the lease updates should be queued. This is the case when the
1326 // server is in the communication-recovery state. Queued lease updates may
1327 // be sent when the communication is re-established.
1328 if (shouldQueueLeaseUpdates(conf)) {
1329 for (auto const& l : *deleted_leases) {
1330 // If a released lease is preserved in the database send the lease
1331 // update to the partner. Otherwise, delete the lease.
1332 if (l->state_ == Lease4::STATE_RELEASED) {
1333 lease_update_backlog_.push(LeaseUpdateBacklog::ADD, l);
1334 } else {
1335 lease_update_backlog_.push(LeaseUpdateBacklog::DELETE, l);
1336 }
1337 }
1338
1339 // Lease updates for new allocations and updated leases.
1340 for (auto const& l : *leases) {
1341 lease_update_backlog_.push(LeaseUpdateBacklog::ADD, l);
1342 }
1343
1344 continue;
1345 }
1346
1347 // Check if the lease update should be sent to the server. If we're in
1348 // the partner-down state we don't send lease updates to the partner.
1349 if (!shouldSendLeaseUpdates(conf)) {
1350 // If we decide to not send the lease updates to an active partner, we
1351 // should make a record of it in the communication state. The partner
1352 // can check if there were any unsent lease updates when he determines
1353 // whether it should synchronize its database or not when it recovers
1354 // from the partner-down state.
1355 if (conf->getRole() != HAConfig::PeerConfig::BACKUP) {
1356 communication_state_->increaseUnsentUpdateCount();
1357 }
1358 continue;
1359 }
1360
1361 // If we're contacting a backup server from which we don't expect a
1362 // response prior to responding to the DHCP client we don't count
1363 // it.
1364 if (config_->amWaitingBackupAck() || (conf->getRole() != HAConfig::PeerConfig::BACKUP)) {
1365 ++sent_num;
1366 }
1367
1368 // Send new/updated leases and deleted leases in one command.
1369 asyncSendLeaseUpdate(query, conf, CommandCreator::createLease6BulkApply(leases, deleted_leases),
1370 parking_lot);
1371 }
1372
1373 return (sent_num);
1374}
1375
1376 template<typename QueryPtrType>
1377 bool
1378 HAService::leaseUpdateComplete(QueryPtrType& query,
1379 const ParkingLotHandlePtr& parking_lot) {
1380 if (MultiThreadingMgr::instance().getMode()) {
1381 std::lock_guard<std::mutex> lock(mutex_);
1382 return (leaseUpdateCompleteInternal(query, parking_lot));
1383 } else {
1384 return (leaseUpdateCompleteInternal(query, parking_lot));
1385 }
1386}
1387
1388 template<typename QueryPtrType>
1389 bool
1390HAService::leaseUpdateCompleteInternal(QueryPtrType& query,
1391 const ParkingLotHandlePtr& parking_lot) {
1392 auto it = pending_requests_.find(query);
1393
1394 // If there are no more pending requests for this query, let's unpark
1395 // the DHCP packet.
1396 if (it == pending_requests_.end() || (--pending_requests_[query] <= 0)) {
1397 if (parking_lot) {
1398 parking_lot->unpark(query);
1399 }
1400
1401 // If we have unparked the packet we can clear pending requests for
1402 // this query.
1403 if (it != pending_requests_.end()) {
1404 pending_requests_.erase(it);
1405 }
1406 return (true);
1407 }
1408 return (false);
1409}
1410
1411 template<typename QueryPtrType>
1412 void
1413 HAService::updatePendingRequest(QueryPtrType& query) {
1414 if (MultiThreadingMgr::instance().getMode()) {
1415 std::lock_guard<std::mutex> lock(mutex_);
1416 updatePendingRequestInternal(query);
1417 } else {
1418 updatePendingRequestInternal(query);
1419 }
1420}
1421
1422 template<typename QueryPtrType>
1423 void
1424HAService::updatePendingRequestInternal(QueryPtrType& query) {
1425 if (pending_requests_.count(query) == 0) {
1426 pending_requests_[query] = 1;
1427 } else {
1428 ++pending_requests_[query];
1429 }
1430}
1431
1432 template<typename QueryPtrType>
1433 void
1434 HAService::asyncSendLeaseUpdate(const QueryPtrType& query,
1435 const HAConfig::PeerConfigPtr& config,
1436 const ConstElementPtr& command,
1437 const ParkingLotHandlePtr& parking_lot) {
1438 // Create HTTP/1.1 request including our command.
1439 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1440 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
1441 HostHttpHeader(config->getUrl().getStrippedHostname()));
1442 config->addBasicAuthHttpHeader(request);
1443 request->setBodyAsJson(command);
1444 request->finalize();
1445
1446 // Response object should also be created because the HTTP client needs
1447 // to know the type of the expected response.
1448 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1449
1450 // When possible we prefer to pass weak pointers to the queries, rather
1451 // than shared pointers, to avoid memory leaks in case cross reference
1452 // between the pointers.
1453 boost::weak_ptr<typename QueryPtrType::element_type> weak_query(query);
1454
1455 // Schedule asynchronous HTTP request.
1456 client_->asyncSendRequest(config->getUrl(), config->getTlsContext(),
1457 request, response,
1458 [this, weak_query, parking_lot, config]
1459 (const boost::system::error_code& ec,
1460 const HttpResponsePtr& response,
1461 const std::string& error_str) {
1462 // Get the shared pointer of the query. The server should keep the
1463 // pointer to the query and then park it. Therefore, we don't really
1464 // expect it to be null. If it is null, something is really wrong.
1465 QueryPtrType query = weak_query.lock();
1466 if (!query) {
1467 isc_throw(Unexpected, "query is null while receiving response from"
1468 " HA peer. This is programmatic error");
1469 }
1470
1471 // There are four possible groups of errors during the lease update.
1472 // One is the IO error causing issues in communication with the peer.
1473 // Another one is an HTTP parsing error. The third type occurs when
1474 // the partner receives the command but it is invalid or there is
1475 // an internal processing error. Finally, the forth type is when the
1476 // conflict status code is returned in the response indicating that
1477 // the lease update does not match the partner's configuration.
1478
1479 bool lease_update_success = true;
1480 bool lease_update_conflict = false;
1481
1482 // Handle first two groups of errors.
1483 if (ec || !error_str.empty()) {
1484 LOG_WARN(ha_logger, HA_LEASE_UPDATE_COMMUNICATIONS_FAILED)
1485 .arg(config_->getThisServerName())
1486 .arg(query->getLabel())
1487 .arg(config->getLogLabel())
1488 .arg(ec ? ec.message() : error_str);
1489
1490 // Communication error, so let's drop parked packet. The DHCP
1491 // response will not be sent.
1492 lease_update_success = false;
1493
1494 } else {
1495
1496 try {
1497 int rcode = 0;
1498 auto args = verifyAsyncResponse(response, rcode);
1499 // In the v6 case the server may return a list of failed lease
1500 // updates and we should log them.
1501 logFailedLeaseUpdates(query, args);
1502
1503 } catch (const ConflictError& ex) {
1504 // Handle forth group of errors.
1505 lease_update_conflict = true;
1506 lease_update_success = false;
1507 communication_state_->reportRejectedLeaseUpdate(query);
1508
1509 LOG_WARN(ha_logger, HA_LEASE_UPDATE_CONFLICT)
1510 .arg(config_->getThisServerName())
1511 .arg(query->getLabel())
1512 .arg(config->getLogLabel())
1513 .arg(ex.what());
1514
1515 } catch (const std::exception& ex) {
1516 // Handle third group of errors.
1517 LOG_WARN(ha_logger, HA_LEASE_UPDATE_FAILED)
1518 .arg(config_->getThisServerName())
1519 .arg(query->getLabel())
1520 .arg(config->getLogLabel())
1521 .arg(ex.what());
1522
1523 // Error while doing an update. The DHCP response will not be sent.
1524 lease_update_success = false;
1525 }
1526 }
1527
1528 // We don't care about the result of the lease update to the backup server.
1529 // It is a best effort update.
1530 if (config->getRole() != HAConfig::PeerConfig::BACKUP) {
1531 // If the lease update was unsuccessful we may need to set the partner
1532 // state as unavailable.
1533 if (!lease_update_success) {
1534 // Do not set it as unavailable if it was a conflict because the
1535 // partner actually responded.
1536 if (!lease_update_conflict) {
1537 // If we were unable to communicate with the partner we set partner's
1538 // state as unavailable.
1539 communication_state_->setPartnerUnavailable();
1540 }
1541 } else {
1542 // Lease update successful and we may need to clear some previously
1543 // rejected lease updates.
1544 communication_state_->reportSuccessfulLeaseUpdate(query);
1545 }
1546 }
1547
1548 // It is possible to configure the server to not wait for a response from
1549 // the backup server before we unpark the packet and respond to the client.
1550 // Here we check if we're dealing with such situation.
1551 if (config_->amWaitingBackupAck() || (config->getRole() != HAConfig::PeerConfig::BACKUP)) {
1552 // We're expecting a response from the backup server or it is not
1553 // a backup server and the lease update was unsuccessful. In such
1554 // case the DHCP exchange fails.
1555 if (!lease_update_success) {
1556 if (parking_lot) {
1557 parking_lot->drop(query);
1558 }
1559 }
1560 } else {
1561 // This was a response from the backup server and we're configured to
1562 // not wait for their acknowledgments, so there is nothing more to do.
1563 return;
1564 }
1565
1566 if (leaseUpdateComplete(query, parking_lot)) {
1567 // If we have finished sending the lease updates we need to run the
1568 // state machine until the state machine finds that additional events
1569 // are required, such as next heartbeat or a lease update. The runModel()
1570 // may transition to another state, schedule asynchronous tasks etc.
1571 // Then it returns control to the DHCP server.
1572 runModel(HA_LEASE_UPDATES_COMPLETE_EVT);
1573 }
1574 },
1575 HttpClient::RequestTimeout(TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST),
1576 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
1577 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
1578 std::bind(&HAService::clientCloseHandler, this, ph::_1)
1579 );
1580
1581 // The number of pending requests is the number of requests for which we
1582 // expect an acknowledgment prior to responding to the DHCP clients. If
1583 // we're configured to wait for the acks from the backups or it is not
1584 // a backup increase the number of pending requests.
1585 if (config_->amWaitingBackupAck() || (config->getRole() != HAConfig::PeerConfig::BACKUP)) {
1586 // Request scheduled, so update the request counters for the query.
1587 updatePendingRequest(query);
1588 }
1589}
1590
1591 bool
1592 HAService::shouldSendLeaseUpdates(const HAConfig::PeerConfigPtr& peer_config) const {
1593 // Never send lease updates if they are administratively disabled.
1594 if (!config_->amSendingLeaseUpdates()) {
1595 return (false);
1596 }
1597
1598 // Always send updates to the backup server.
1599 if (peer_config->getRole() == HAConfig::PeerConfig::BACKUP) {
1600 return (true);
1601 }
1602
1603 // Never send updates if this is a backup server.
1604 if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
1605 return (false);
1606 }
1607
1608 // In other case, whether we send lease updates or not depends on our
1609 // state.
1610 switch (getCurrState()) {
1611 case HA_HOT_STANDBY_ST:
1612 case HA_LOAD_BALANCING_ST:
1613 case HA_PARTNER_IN_MAINTENANCE_ST:
1614 return (true);
1615
1616 default:
1617 ;
1618 }
1619
1620 return (false);
1621}
1622
1623 bool
1624 HAService::shouldQueueLeaseUpdates(const HAConfig::PeerConfigPtr& peer_config) const {
1625 if (!config_->amSendingLeaseUpdates()) {
1626 return (false);
1627 }
1628
1629 if (peer_config->getRole() == HAConfig::PeerConfig::BACKUP) {
1630 return (false);
1631 }
1632
1633 return (getCurrState() == HA_COMMUNICATION_RECOVERY_ST);
1634}
1635
1636 void
1637 HAService::logFailedLeaseUpdates(const PktPtr& query,
1638 const ConstElementPtr& args) const {
1639 // If there are no arguments, it means that the update was successful.
1640 if (!args || (args->getType() != Element::map)) {
1641 return;
1642 }
1643
1644 // Instead of duplicating the code between the failed-deleted-leases and
1645 // failed-leases, let's just have one function that does it for both.
1646 auto log_proc = [](const PktPtr query, const ConstElementPtr& args,
1647 const std::string& param_name, const log::MessageID& mesid) {
1648
1649 // Check if there are any failed leases.
1650 auto failed_leases = args->get(param_name);
1651
1652 // The failed leases must be a list.
1653 if (failed_leases && (failed_leases->getType() == Element::list)) {
1654 // Go over the failed leases and log each of them.
1655 for (unsigned i = 0; i < failed_leases->size(); ++i) {
1656 auto lease = failed_leases->get(i);
1657 if (lease->getType() == Element::map) {
1658
1659 // ip-address
1660 auto ip_address = lease->get("ip-address");
1661
1662 // lease type
1663 auto lease_type = lease->get("type");
1664
1665 // error-message
1666 auto error_message = lease->get("error-message");
1667
1668 LOG_INFO(ha_logger, mesid)
1669 .arg(query->getLabel())
1670 .arg(lease_type && (lease_type->getType() == Element::string) ?
1671 lease_type->stringValue() : "(unknown)")
1672 .arg(ip_address && (ip_address->getType() == Element::string) ?
1673 ip_address->stringValue() : "(unknown)")
1674 .arg(error_message && (error_message->getType() == Element::string) ?
1675 error_message->stringValue() : "(unknown)");
1676 }
1677 }
1678 }
1679 };
1680
1681 // Process "failed-deleted-leases"
1682 log_proc(query, args, "failed-deleted-leases", HA_LEASE_UPDATE_DELETE_FAILED_ON_PEER);
1683
1684 // Process "failed-leases".
1685 log_proc(query, args, "failed-leases", HA_LEASE_UPDATE_CREATE_UPDATE_FAILED_ON_PEER);
1686}
1687
1688 ConstElementPtr
1689 HAService::processStatusGet() const {
1690 ElementPtr ha_servers = Element::createMap();
1691
1692 // Local part
1693 ElementPtr local = Element::createMap();
1694 HAConfig::PeerConfig::Role role;
1695 role = config_->getThisServerConfig()->getRole();
1696 std::string role_txt = HAConfig::PeerConfig::roleToString(role);
1697 local->set("role", Element::create(role_txt));
1698 int state = getCurrState();
1699 try {
1700 local->set("state", Element::create(stateToString(state)));
1701
1702 } catch (...) {
1703 // Empty string on error.
1704 local->set("state", Element::create(std::string()));
1705 }
1706 std::set<std::string> scopes = query_filter_.getServedScopes();
1707 ElementPtr list = Element::createList();
1708 for (auto const& scope : scopes) {
1709 list->add(Element::create(scope));
1710 }
1711 local->set("scopes", list);
1712 local->set("server-name", Element::create(config_->getThisServerName()));
1713 auto const my_time(communication_state_->getMyTimeAtSkew());
1714 if (my_time.is_not_a_date_time()) {
1715 local->set("system-time", Element::create());
1716 } else {
1717 local->set("system-time", Element::create(ptimeToText(my_time, 0)));
1718 }
1719 ha_servers->set("local", local);
1720
1721 // Do not include remote server information if this is a backup server or
1722 // we're in the passive-backup mode.
1723 if ((config_->getHAMode() == HAConfig::PASSIVE_BACKUP) ||
1724 (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP)) {
1725 return (ha_servers);
1726 }
1727
1728 // Remote part
1729 ElementPtr remote = communication_state_->getReport();
1730
1731 try {
1732 role = config_->getFailoverPeerConfig()->getRole();
1733 role_txt = HAConfig::PeerConfig::roleToString(role);
1734 remote->set("role", Element::create(role_txt));
1735
1736 } catch (...) {
1737 remote->set("role", Element::create(std::string()));
1738 }
1739 remote->set("server-name", Element::create(config_->getFailoverPeerConfig()->getName()));
1740 ha_servers->set("remote", remote);
1741
1742 return (ha_servers);
1743}
1744
1745 ConstElementPtr
1746 HAService::processHeartbeat() {
1747 ElementPtr arguments = Element::createMap();
1748 std::string state_label = getState(getCurrState())->getLabel();
1749 arguments->set("state", Element::create(state_label));
1750
1751 std::string date_time = HttpDateTime().rfc1123Format();
1752 arguments->set("date-time", Element::create(date_time));
1753
1754 auto scopes = query_filter_.getServedScopes();
1755 ElementPtr scopes_list = Element::createList();
1756 for (auto const& scope : scopes) {
1757 scopes_list->add(Element::create(scope));
1758 }
1759 arguments->set("scopes", scopes_list);
1760
1761 arguments->set("unsent-update-count",
1762 Element::create(static_cast<int64_t>(communication_state_->getUnsentUpdateCount())));
1763
1764 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA peer status returned.",
1765 arguments));
1766}
1767
1768 ConstElementPtr
1769 HAService::processHAReset() {
1770 if (getCurrState() == HA_WAITING_ST) {
1771 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine already in WAITING state."));
1772 }
1773 verboseTransition(HA_WAITING_ST);
1774 runModel(NOP_EVT);
1775 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine reset."));
1776}
1777
1778 void
1779 HAService::asyncSendHeartbeat() {
1780 HAConfig::PeerConfigPtr partner_config = config_->getFailoverPeerConfig();
1781
1782 // If the sync_complete_notified_ is true it means that the partner
1783 // notified us that it had completed lease database synchronization.
1784 // We confirm that the partner is operational by sending the heartbeat
1785 // to it. Regardless if the partner responds to our heartbeats or not,
1786 // we should clear this flag. But, since we need the current value in
1787 // the async call handler, we save it in the local variable before
1788 // clearing it.
1789 bool sync_complete_notified = sync_complete_notified_;
1790 sync_complete_notified_ = false;
1791
1792 // Create HTTP/1.1 request including our command.
1793 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1794 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
1795 HostHttpHeader(partner_config->getUrl().getStrippedHostname()));
1796 partner_config->addBasicAuthHttpHeader(request);
1797 request->setBodyAsJson(CommandCreator::createHeartbeat(config_->getThisServerName(),
1798 server_type_));
1799 request->finalize();
1800
1801 // Response object should also be created because the HTTP client needs
1802 // to know the type of the expected response.
1803 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1804
1805 // Schedule asynchronous HTTP request.
1806 client_->asyncSendRequest(partner_config->getUrl(),
1807 partner_config->getTlsContext(),
1808 request, response,
1809 [this, partner_config, sync_complete_notified]
1810 (const boost::system::error_code& ec,
1811 const HttpResponsePtr& response,
1812 const std::string& error_str) {
1813
1814 // There are three possible groups of errors during the heartbeat.
1815 // One is the IO error causing issues in communication with the peer.
1816 // Another one is an HTTP parsing error. The last type of error is
1817 // when non-success error code is returned in the response carried
1818 // in the HTTP message or if the JSON response is otherwise broken.
1819
1820 bool heartbeat_success = true;
1821
1822 // Handle first two groups of errors.
1823 if (ec || !error_str.empty()) {
1824 LOG_WARN(ha_logger, HA_HEARTBEAT_COMMUNICATIONS_FAILED)
1825 .arg(config_->getThisServerName())
1826 .arg(partner_config->getLogLabel())
1827 .arg(ec ? ec.message() : error_str);
1828 heartbeat_success = false;
1829
1830 } else {
1831
1832 // Handle third group of errors.
1833 try {
1834 // Response must contain arguments and the arguments must
1835 // be a map.
1836 int rcode = 0;
1837 ConstElementPtr args = verifyAsyncResponse(response, rcode);
1838 if (!args || args->getType() != Element::map) {
1839 isc_throw(CtrlChannelError, "returned arguments in the response"
1840 " must be a map");
1841 }
1842 // Response must include partner's state.
1843 ConstElementPtr state = args->get("state");
1844 if (!state || state->getType() != Element::string) {
1845 isc_throw(CtrlChannelError, "server state not returned in response"
1846 " to a ha-heartbeat command or it is not a string");
1847 }
1848 // Remember the partner's state. This may throw if the returned
1849 // state is invalid.
1850 communication_state_->setPartnerState(state->stringValue());
1851
1852 ConstElementPtr date_time = args->get("date-time");
1853 if (!date_time || date_time->getType() != Element::string) {
1854 isc_throw(CtrlChannelError, "date-time not returned in response"
1855 " to a ha-heartbeat command or it is not a string");
1856 }
1857 // Note the time returned by the partner to calculate the clock skew.
1858 communication_state_->setPartnerTime(date_time->stringValue());
1859
1860 // Remember the scopes served by the partner.
1861 try {
1862 auto scopes = args->get("scopes");
1863 communication_state_->setPartnerScopes(scopes);
1864
1865 } catch (...) {
1866 // We don't want to fail if the scopes are missing because
1867 // this would be incompatible with old HA hook library
1868 // versions. We may make it mandatory one day, but during
1869 // upgrades of existing HA setup it would be a real issue
1870 // if we failed here.
1871 }
1872
1873 // unsent-update-count was not present in earlier HA versions.
1874 // Let's check if the partner has sent the parameter. We initialized
1875 // the counter to 0, and it remains 0 if the partner doesn't send it.
1876 // It effectively means that we don't track partner's unsent updates
1877 // as in the earlier HA versions.
1878 auto unsent_update_count = args->get("unsent-update-count");
1879 if (unsent_update_count) {
1880 if (unsent_update_count->getType() != Element::integer) {
1881 isc_throw(CtrlChannelError, "unsent-update-count returned in"
1882 " the ha-heartbeat response is not an integer");
1883 }
1884 communication_state_->setPartnerUnsentUpdateCount(static_cast<uint64_t>
1885 (unsent_update_count->intValue()));
1886 }
1887
1888 } catch (const std::exception& ex) {
1889 LOG_WARN(ha_logger, HA_HEARTBEAT_FAILED)
1890 .arg(config_->getThisServerName())
1891 .arg(partner_config->getLogLabel())
1892 .arg(ex.what());
1893 heartbeat_success = false;
1894 }
1895 }
1896
1897 // If heartbeat was successful, let's mark the connection with the
1898 // peer as healthy.
1899 if (heartbeat_success) {
1900 communication_state_->poke();
1901
1902 } else {
1903 // We were unable to retrieve partner's state, so let's mark it
1904 // as unavailable.
1905 communication_state_->setPartnerUnavailable();
1906 // Log if the communication is interrupted.
1907 if (communication_state_->isCommunicationInterrupted()) {
1908 LOG_WARN(ha_logger, HA_COMMUNICATION_INTERRUPTED)
1909 .arg(config_->getThisServerName())
1910 .arg(partner_config->getName());
1911 }
1912 }
1913
1914 startHeartbeat();
1915 // Even though the partner notified us about the synchronization completion,
1916 // we still can't communicate with the partner. Let's continue serving
1917 // the clients until the link is fixed.
1918 if (sync_complete_notified && !heartbeat_success) {
1919 postNextEvent(HA_SYNCED_PARTNER_UNAVAILABLE_EVT);
1920 }
1921 // Whatever the result of the heartbeat was, the state machine needs
1922 // to react to this. Let's run the state machine until the state machine
1923 // finds that some new events are required, i.e. next heartbeat or
1924 // lease update. The runModel() may transition to another state, schedule
1925 // asynchronous tasks etc. Then it returns control to the DHCP server.
1926 runModel(HA_HEARTBEAT_COMPLETE_EVT);
1927 },
1928 HttpClient::RequestTimeout(TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST),
1929 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
1930 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
1931 std::bind(&HAService::clientCloseHandler, this, ph::_1)
1932 );
1933}
1934
1935 void
1936 HAService::scheduleHeartbeat() {
1937 if (!communication_state_->isHeartbeatRunning()) {
1938 startHeartbeat();
1939 }
1940}
1941
1942 void
1943 HAService::startHeartbeat() {
1944 if (config_->getHeartbeatDelay() > 0) {
1945 communication_state_->startHeartbeat(config_->getHeartbeatDelay(),
1946 std::bind(&HAService::asyncSendHeartbeat,
1947 this));
1948 }
1949}
1950
1951 void
1952 HAService::asyncDisableDHCPService(HttpClient& http_client,
1953 const HAConfig::PeerConfigPtr& remote_config,
1954 const unsigned int max_period,
1955 PostRequestCallback post_request_action) {
1956 // Create HTTP/1.1 request including our command.
1957 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
1958 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
1959 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
1960
1961 remote_config->addBasicAuthHttpHeader(request);
1962 request->setBodyAsJson(CommandCreator::createDHCPDisable(getRemoteOrigin(),
1963 max_period,
1964 server_type_));
1965 request->finalize();
1966
1967 // Response object should also be created because the HTTP client needs
1968 // to know the type of the expected response.
1969 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
1970
1971 // Schedule asynchronous HTTP request.
1972 http_client.asyncSendRequest(remote_config->getUrl(),
1973 remote_config->getTlsContext(),
1974 request, response,
1975 [this, remote_config, post_request_action]
1976 (const boost::system::error_code& ec,
1977 const HttpResponsePtr& response,
1978 const std::string& error_str) {
1979
1980 // There are three possible groups of errors during the heartbeat.
1981 // One is the IO error causing issues in communication with the peer.
1982 // Another one is an HTTP parsing error. The last type of error is
1983 // when non-success error code is returned in the response carried
1984 // in the HTTP message or if the JSON response is otherwise broken.
1985
1986 int rcode = 0;
1987 std::string error_message;
1988
1989 // Handle first two groups of errors.
1990 if (ec || !error_str.empty()) {
1991 error_message = (ec ? ec.message() : error_str);
1992 LOG_ERROR(ha_logger, HA_DHCP_DISABLE_COMMUNICATIONS_FAILED)
1993 .arg(config_->getThisServerName())
1994 .arg(remote_config->getLogLabel())
1995 .arg(error_message);
1996
1997 } else {
1998
1999 // Handle third group of errors.
2000 try {
2001 static_cast<void>(verifyAsyncResponse(response, rcode));
2002
2003 } catch (const std::exception& ex) {
2004 error_message = ex.what();
2005 LOG_ERROR(ha_logger, HA_DHCP_DISABLE_FAILED)
2006 .arg(config_->getThisServerName())
2007 .arg(remote_config->getLogLabel())
2008 .arg(error_message);
2009 }
2010 }
2011
2012 // If there was an error communicating with the partner, mark the
2013 // partner as unavailable.
2014 if (!error_message.empty()) {
2015 communication_state_->setPartnerUnavailable();
2016 }
2017
2018 // Invoke post request action if it was specified.
2019 if (post_request_action) {
2020 post_request_action(error_message.empty(),
2021 error_message,
2022 rcode);
2023 }
2024 },
2025 HttpClient::RequestTimeout(TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST),
2026 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2027 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2028 std::bind(&HAService::clientCloseHandler, this, ph::_1)
2029 );
2030}
2031
2032 void
2033 HAService::asyncEnableDHCPService(HttpClient& http_client,
2034 const HAConfig::PeerConfigPtr& remote_config,
2035 PostRequestCallback post_request_action) {
2036 // Create HTTP/1.1 request including our command.
2037 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2038 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2039 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2040 remote_config->addBasicAuthHttpHeader(request);
2041 request->setBodyAsJson(CommandCreator::createDHCPEnable(getRemoteOrigin(),
2042 server_type_));
2043 request->finalize();
2044
2045 // Response object should also be created because the HTTP client needs
2046 // to know the type of the expected response.
2047 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2048
2049 // Schedule asynchronous HTTP request.
2050 http_client.asyncSendRequest(remote_config->getUrl(),
2051 remote_config->getTlsContext(),
2052 request, response,
2053 [this, remote_config, post_request_action]
2054 (const boost::system::error_code& ec,
2055 const HttpResponsePtr& response,
2056 const std::string& error_str) {
2057
2058 // There are three possible groups of errors during the heartbeat.
2059 // One is the IO error causing issues in communication with the peer.
2060 // Another one is an HTTP parsing error. The last type of error is
2061 // when non-success error code is returned in the response carried
2062 // in the HTTP message or if the JSON response is otherwise broken.
2063
2064 int rcode = 0;
2065 std::string error_message;
2066
2067 // Handle first two groups of errors.
2068 if (ec || !error_str.empty()) {
2069 error_message = (ec ? ec.message() : error_str);
2070 LOG_ERROR(ha_logger, HA_DHCP_ENABLE_COMMUNICATIONS_FAILED)
2071 .arg(config_->getThisServerName())
2072 .arg(remote_config->getLogLabel())
2073 .arg(error_message);
2074
2075 } else {
2076
2077 // Handle third group of errors.
2078 try {
2079 static_cast<void>(verifyAsyncResponse(response, rcode));
2080
2081 } catch (const std::exception& ex) {
2082 error_message = ex.what();
2083 LOG_ERROR(ha_logger, HA_DHCP_ENABLE_FAILED)
2084 .arg(config_->getThisServerName())
2085 .arg(remote_config->getLogLabel())
2086 .arg(error_message);
2087 }
2088 }
2089
2090 // If there was an error communicating with the partner, mark the
2091 // partner as unavailable.
2092 if (!error_message.empty()) {
2093 communication_state_->setPartnerUnavailable();
2094 }
2095
2096 // Invoke post request action if it was specified.
2097 if (post_request_action) {
2098 post_request_action(error_message.empty(),
2099 error_message,
2100 rcode);
2101 }
2102 },
2103 HttpClient::RequestTimeout(TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST),
2104 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2105 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2106 std::bind(&HAService::clientCloseHandler, this, ph::_1)
2107 );
2108}
2109
2110 void
2111 HAService::localDisableDHCPService() {
2112 network_state_->disableService(getLocalOrigin());
2113}
2114
2115 void
2116 HAService::localEnableDHCPService() {
2117 network_state_->enableService(getLocalOrigin());
2118}
2119
2120 void
2121 HAService::asyncSyncLeases() {
2122 PostSyncCallback null_action;
2123
2124 // Timeout is configured in milliseconds. Need to convert to seconds.
2125 unsigned int dhcp_disable_timeout =
2126 static_cast<unsigned int>(config_->getSyncTimeout() / 1000);
2127 if (dhcp_disable_timeout == 0) {
2128 // Ensure that we always use at least 1 second timeout.
2129 dhcp_disable_timeout = 1;
2130 }
2131
2132 lease_sync_filter_.apply();
2133 asyncSyncLeases(*client_, config_->getFailoverPeerConfig(),
2134 dhcp_disable_timeout, LeasePtr(), null_action);
2135}
2136
2137 void
2138 HAService::asyncSyncLeases(http::HttpClient& http_client,
2139 const HAConfig::PeerConfigPtr& remote_config,
2140 const unsigned int max_period,
2141 const dhcp::LeasePtr& last_lease,
2142 PostSyncCallback post_sync_action,
2143 const bool dhcp_disabled) {
2144 // Synchronization starts with a command to disable DHCP service of the
2145 // peer from which we're fetching leases. We don't want the other server
2146 // to allocate new leases while we fetch from it. The DHCP service will
2147 // be disabled for a certain amount of time and will be automatically
2148 // re-enabled if we die during the synchronization.
2149 asyncDisableDHCPService(http_client, remote_config, max_period,
2150 [this, &http_client, remote_config, max_period, last_lease,
2151 post_sync_action, dhcp_disabled]
2152 (const bool success, const std::string& error_message, const int) {
2153
2154 // If we have successfully disabled the DHCP service on the peer,
2155 // we can start fetching the leases.
2156 if (success) {
2157 // The last argument indicates that disabling the DHCP
2158 // service on the partner server was successful.
2159 asyncSyncLeasesInternal(http_client, remote_config, max_period,
2160 last_lease, post_sync_action, true);
2161
2162 } else {
2163 post_sync_action(success, error_message, dhcp_disabled);
2164 }
2165 });
2166}
2167
2168 void
2169 HAService::asyncSyncLeasesInternal(http::HttpClient& http_client,
2170 const HAConfig::PeerConfigPtr& remote_config,
2171 const unsigned int max_period,
2172 const dhcp::LeasePtr& last_lease,
2173 PostSyncCallback post_sync_action,
2174 const bool dhcp_disabled) {
2175 // Create HTTP/1.1 request including our command.
2176 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2177 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2178 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2179 remote_config->addBasicAuthHttpHeader(request);
2180 if (server_type_ == HAServerType::DHCPv4) {
2181 request->setBodyAsJson(CommandCreator::createLease4GetPage(
2182 boost::dynamic_pointer_cast<Lease4>(last_lease), config_->getSyncPageLimit()));
2183
2184 } else {
2185 request->setBodyAsJson(CommandCreator::createLease6GetPage(
2186 boost::dynamic_pointer_cast<Lease6>(last_lease), config_->getSyncPageLimit()));
2187 }
2188 request->finalize();
2189
2190 // Response object should also be created because the HTTP client needs
2191 // to know the type of the expected response.
2192 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2193
2194 // Schedule asynchronous HTTP request.
2195 http_client.asyncSendRequest(remote_config->getUrl(),
2196 remote_config->getTlsContext(),
2197 request, response,
2198 [this, remote_config, post_sync_action, &http_client, max_period, dhcp_disabled]
2199 (const boost::system::error_code& ec,
2200 const HttpResponsePtr& response,
2201 const std::string& error_str) {
2202
2203 // Holds last lease received on the page of leases. If the last
2204 // page was hit, this value remains null.
2205 LeasePtr last_lease;
2206
2207 // There are three possible groups of errors during the heartbeat.
2208 // One is the IO error causing issues in communication with the peer.
2209 // Another one is an HTTP parsing error. The last type of error is
2210 // when non-success error code is returned in the response carried
2211 // in the HTTP message or if the JSON response is otherwise broken.
2212
2213 std::string error_message;
2214
2215 // Handle first two groups of errors.
2216 if (ec || !error_str.empty()) {
2217 error_message = (ec ? ec.message() : error_str);
2218 LOG_ERROR(ha_logger, HA_LEASES_SYNC_COMMUNICATIONS_FAILED)
2219 .arg(config_->getThisServerName())
2220 .arg(remote_config->getLogLabel())
2221 .arg(error_message);
2222
2223 } else {
2224 // Handle third group of errors.
2225 try {
2226 int rcode = 0;
2227 ConstElementPtr args = verifyAsyncResponse(response, rcode);
2228
2229 // Arguments must be a map.
2230 if (args && (args->getType() != Element::map)) {
2231 isc_throw(CtrlChannelError,
2232 "arguments in the received response must be a map");
2233 }
2234
2235 ConstElementPtr leases = args->get("leases");
2236 if (!leases || (leases->getType() != Element::list)) {
2237 isc_throw(CtrlChannelError,
2238 "server response does not contain leases argument or this"
2239 " argument is not a list");
2240 }
2241
2242 // Iterate over the leases and update the database as appropriate.
2243 auto const& leases_element = leases->listValue();
2244
2245 LOG_INFO(ha_logger, HA_LEASES_SYNC_LEASE_PAGE_RECEIVED)
2246 .arg(config_->getThisServerName())
2247 .arg(leases_element.size())
2248 .arg(remote_config->getLogLabel());
2249
2250 // Count actually applied leases.
2251 uint64_t applied_lease_count = 0;
2252 for (auto l = leases_element.begin(); l != leases_element.end(); ++l) {
2253 try {
2254
2255 if (server_type_ == HAServerType::DHCPv4) {
2256 Lease4Ptr lease = Lease4::fromElement(*l);
2257
2258 // If we're not on the last page and we're processing final lease on
2259 // this page, let's record the lease as input to the next
2260 // lease4-get-page command.
2261 if ((leases_element.size() >= config_->getSyncPageLimit()) &&
2262 (l + 1 == leases_element.end())) {
2263 last_lease = boost::dynamic_pointer_cast<Lease>(lease);
2264 }
2265
2266 if (!lease_sync_filter_.shouldSync(lease)) {
2267 continue;
2268 }
2269
2270 // Check if there is such lease in the database already.
2271 Lease4Ptr existing_lease = LeaseMgrFactory::instance().getLease4(lease->addr_);
2272 if (!existing_lease) {
2273 // There is no such lease, so let's add it.
2274 LeaseMgrFactory::instance().addLease(lease);
2275 ++applied_lease_count;
2276
2277 } else if (existing_lease->cltt_ < lease->cltt_) {
2278 // If the existing lease is older than the fetched lease, update
2279 // the lease in our local database.
2280 // Update lease current expiration time with value received from the
2281 // database. Some database backends reject operations on the lease if
2282 // the current expiration time value does not match what is stored.
2283 Lease::syncCurrentExpirationTime(*existing_lease, *lease);
2284 LeaseMgrFactory::instance().updateLease4(lease);
2285 ++applied_lease_count;
2286
2287 } else {
2288 LOG_DEBUG(ha_logger, DBGLVL_TRACE_BASIC, HA_LEASE_SYNC_STALE_LEASE4_SKIP)
2289 .arg(config_->getThisServerName())
2290 .arg(lease->addr_.toText())
2291 .arg(lease->subnet_id_);
2292 }
2293
2294 } else {
2295 Lease6Ptr lease = Lease6::fromElement(*l);
2296
2297 // If we're not on the last page and we're processing final lease on
2298 // this page, let's record the lease as input to the next
2299 // lease6-get-page command.
2300 if ((leases_element.size() >= config_->getSyncPageLimit()) &&
2301 (l + 1 == leases_element.end())) {
2302 last_lease = boost::dynamic_pointer_cast<Lease>(lease);
2303 }
2304
2305 if (!lease_sync_filter_.shouldSync(lease)) {
2306 continue;
2307 }
2308
2309 // Check if there is such lease in the database already.
2310 Lease6Ptr existing_lease = LeaseMgrFactory::instance().getLease6(lease->type_,
2311 lease->addr_);
2312 if (!existing_lease) {
2313 // There is no such lease, so let's add it.
2314 LeaseMgrFactory::instance().addLease(lease);
2315 ++applied_lease_count;
2316
2317 } else if (existing_lease->cltt_ < lease->cltt_) {
2318 // If the existing lease is older than the fetched lease, update
2319 // the lease in our local database.
2320 // Update lease current expiration time with value received from the
2321 // database. Some database backends reject operations on the lease if
2322 // the current expiration time value does not match what is stored.
2323 Lease::syncCurrentExpirationTime(*existing_lease, *lease);
2324 LeaseMgrFactory::instance().updateLease6(lease);
2325 ++applied_lease_count;
2326
2327 } else {
2328 LOG_DEBUG(ha_logger, DBGLVL_TRACE_BASIC, HA_LEASE_SYNC_STALE_LEASE6_SKIP)
2329 .arg(config_->getThisServerName())
2330 .arg(lease->addr_.toText())
2331 .arg(lease->subnet_id_);
2332 }
2333 }
2334
2335 } catch (const std::exception& ex) {
2336 LOG_WARN(ha_logger, HA_LEASE_SYNC_FAILED)
2337 .arg(config_->getThisServerName())
2338 .arg((*l)->str())
2339 .arg(ex.what());
2340 }
2341 }
2342
2343 LOG_INFO(ha_logger, HA_LEASES_SYNC_APPLIED_LEASES)
2344 .arg(config_->getThisServerName())
2345 .arg(applied_lease_count);
2346
2347 } catch (const std::exception& ex) {
2348 error_message = ex.what();
2349 LOG_ERROR(ha_logger, HA_LEASES_SYNC_FAILED)
2350 .arg(config_->getThisServerName())
2351 .arg(remote_config->getLogLabel())
2352 .arg(error_message);
2353 }
2354 }
2355
2356 // If there was an error communicating with the partner, mark the
2357 // partner as unavailable.
2358 if (!error_message.empty()) {
2359 communication_state_->setPartnerUnavailable();
2360
2361 } else if (last_lease) {
2362 // This indicates that there are more leases to be fetched.
2363 // Therefore, we have to send another leaseX-get-page command.
2364 asyncSyncLeases(http_client, remote_config, max_period, last_lease,
2365 post_sync_action, dhcp_disabled);
2366 return;
2367 }
2368
2369 // Invoke post synchronization action if it was specified.
2370 if (post_sync_action) {
2371 post_sync_action(error_message.empty(),
2372 error_message,
2373 dhcp_disabled);
2374 }
2375 },
2376 HttpClient::RequestTimeout(config_->getSyncTimeout()),
2377 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2378 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2379 std::bind(&HAService::clientCloseHandler, this, ph::_1)
2380 );
2381
2382}
2383
2384 ConstElementPtr
2385 HAService::processSynchronize(const std::string& server_name,
2386 const unsigned int max_period) {
2387 HAConfig::PeerConfigPtr remote_config;
2388 try {
2389 remote_config = config_->getPeerConfig(server_name);
2390 } catch (const std::exception& ex) {
2391 return (createAnswer(CONTROL_RESULT_ERROR, ex.what()));
2392 }
2393 // We must not synchronize with self.
2394 if (remote_config->getName() == config_->getThisServerName()) {
2395 return (createAnswer(CONTROL_RESULT_ERROR, "'" + remote_config->getName()
2396 + "' points to local server but should point to a partner"));
2397 }
2398 std::string answer_message;
2399 int sync_status = synchronize(answer_message, remote_config, max_period);
2400 return (createAnswer(sync_status, answer_message));
2401}
2402
2403 int
2404 HAService::synchronize(std::string& status_message,
2405 const HAConfig::PeerConfigPtr& remote_config,
2406 const unsigned int max_period) {
2407 lease_sync_filter_.apply();
2408
2409 IOServicePtr io_service(new IOService());
2410 HttpClient client(io_service, false);
2411
2412 asyncSyncLeases(client, remote_config, max_period, Lease4Ptr(),
2413 [&](const bool success, const std::string& error_message,
2414 const bool dhcp_disabled) {
2415 // If there was a fatal error while fetching the leases, let's
2416 // log an error message so as it can be included in the response
2417 // to the controlling client.
2418 if (!success) {
2419 status_message = error_message;
2420 }
2421
2422 // Whether or not there was an error while fetching the leases,
2423 // we need to re-enable the DHCP service on the peer if the
2424 // DHCP service was disabled in the course of synchronization.
2425 if (dhcp_disabled) {
2426 // If the synchronization was completed successfully let's
2427 // try to send the ha-sync-complete-notify command to the
2428 // partner.
2429 if (success) {
2430 asyncSyncCompleteNotify(client, remote_config,
2431 [&](const bool success,
2432 const std::string& error_message,
2433 const int rcode) {
2434 // This command may not be supported by the partner when it
2435 // runs an older Kea version. In that case, send the dhcp-enable
2436 // command as in previous Kea version.
2437 if (rcode == CONTROL_RESULT_COMMAND_UNSUPPORTED) {
2438 asyncEnableDHCPService(client, remote_config,
2439 [&](const bool success,
2440 const std::string& error_message,
2441 const int) {
2442 // It is possible that we have already recorded an error
2443 // message while synchronizing the lease database. Don't
2444 // override the existing error message.
2445 if (!success && status_message.empty()) {
2446 status_message = error_message;
2447 }
2448
2449 // The synchronization process is completed, so let's break
2450 // the IO service so as we can return the response to the
2451 // controlling client.
2452 io_service->stop();
2453 });
2454
2455 } else {
2456 // ha-sync-complete-notify command was delivered to the partner.
2457 // The synchronization process ends here.
2458 if (!success && status_message.empty()) {
2459 status_message = error_message;
2460 }
2461
2462 io_service->stop();
2463 }
2464 });
2465
2466 } else {
2467 // Synchronization was unsuccessful. Send the dhcp-enable command to
2468 // re-enable the DHCP service. Note, that we don't send the
2469 // ha-sync-complete-notify command in this case. It is only sent in
2470 // the case when synchronization ends successfully.
2471 asyncEnableDHCPService(client, remote_config,
2472 [&](const bool success,
2473 const std::string& error_message,
2474 const int) {
2475 if (!success && status_message.empty()) {
2476 status_message = error_message;
2477 }
2478
2479 // The synchronization process is completed, so let's break
2480 // the IO service so as we can return the response to the
2481 // controlling client.
2482 io_service->stop();
2483
2484 });
2485 }
2486
2487 } else {
2488 // Also stop IO service if there is no need to enable DHCP
2489 // service.
2490 io_service->stop();
2491 }
2492 });
2493
2494 LOG_INFO(ha_logger, HA_SYNC_START)
2495 .arg(config_->getThisServerName())
2496 .arg(remote_config->getLogLabel());
2497
2498 // Measure duration of the synchronization.
2499 Stopwatch stopwatch;
2500
2501 // Run the IO service until it is stopped by any of the callbacks. This
2502 // makes it synchronous.
2503 io_service->run();
2504
2505 // End measuring duration.
2506 stopwatch.stop();
2507
2508 client.stop();
2509
2510 io_service->stopAndPoll();
2511
2512 // If an error message has been recorded, return an error to the controlling
2513 // client.
2514 if (!status_message.empty()) {
2515 postNextEvent(HA_SYNCING_FAILED_EVT);
2516
2517 LOG_ERROR(ha_logger, HA_SYNC_FAILED)
2518 .arg(config_->getThisServerName())
2519 .arg(remote_config->getLogLabel())
2520 .arg(status_message);
2521
2522 return (CONTROL_RESULT_ERROR);
2523
2524 }
2525
2526 // Everything was fine, so let's return a success.
2527 status_message = "Lease database synchronization complete.";
2528 postNextEvent(HA_SYNCING_SUCCEEDED_EVT);
2529
2530 LOG_INFO(ha_logger, HA_SYNC_SUCCESSFUL)
2531 .arg(config_->getThisServerName())
2532 .arg(remote_config->getLogLabel())
2533 .arg(stopwatch.logFormatLastDuration());
2534
2535 return (CONTROL_RESULT_SUCCESS);
2536}
2537
2538 void
2539 HAService::asyncSendLeaseUpdatesFromBacklog(HttpClient& http_client,
2540 const HAConfig::PeerConfigPtr& config,
2541 PostRequestCallback post_request_action) {
2542 if (lease_update_backlog_.size() == 0) {
2543 post_request_action(true, "", CONTROL_RESULT_SUCCESS);
2544 return;
2545 }
2546
2547 ConstElementPtr command;
2548 if (server_type_ == HAServerType::DHCPv4) {
2549 LeaseUpdateBacklog::OpType op_type;
2550 Lease4Ptr lease = boost::dynamic_pointer_cast<Lease4>(lease_update_backlog_.pop(op_type));
2551 if (op_type == LeaseUpdateBacklog::ADD) {
2552 command = CommandCreator::createLease4Update(*lease);
2553 } else {
2554 command = CommandCreator::createLease4Delete(*lease);
2555 }
2556
2557 } else {
2558 command = CommandCreator::createLease6BulkApply(lease_update_backlog_);
2559 }
2560
2561 // Create HTTP/1.1 request including our command.
2562 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2563 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2564 HostHttpHeader(config->getUrl().getStrippedHostname()));
2565 config->addBasicAuthHttpHeader(request);
2566 request->setBodyAsJson(command);
2567 request->finalize();
2568
2569 // Response object should also be created because the HTTP client needs
2570 // to know the type of the expected response.
2571 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2572
2573 http_client.asyncSendRequest(config->getUrl(), config->getTlsContext(),
2574 request, response,
2575 [this, &http_client, config, post_request_action]
2576 (const boost::system::error_code& ec,
2577 const HttpResponsePtr& response,
2578 const std::string& error_str) {
2579
2580 int rcode = 0;
2581 std::string error_message;
2582
2583 if (ec || !error_str.empty()) {
2584 error_message = (ec ? ec.message() : error_str);
2585 LOG_WARN(ha_logger, HA_LEASES_BACKLOG_COMMUNICATIONS_FAILED)
2586 .arg(config_->getThisServerName())
2587 .arg(config->getLogLabel())
2588 .arg(ec ? ec.message() : error_str);
2589
2590 } else {
2591 // Handle third group of errors.
2592 try {
2593 auto args = verifyAsyncResponse(response, rcode);
2594 } catch (const std::exception& ex) {
2595 error_message = ex.what();
2596 LOG_WARN(ha_logger, HA_LEASES_BACKLOG_FAILED)
2597 .arg(config_->getThisServerName())
2598 .arg(config->getLogLabel())
2599 .arg(ex.what());
2600 }
2601 }
2602
2603 // Recursively send all outstanding lease updates or break when an
2604 // error occurs. In DHCPv6, this is a single iteration because we use
2605 // lease6-bulk-apply, which combines many lease updates in a single
2606 // transaction. In the case of DHCPv4, each update is sent in its own
2607 // transaction.
2608 if (error_message.empty()) {
2609 asyncSendLeaseUpdatesFromBacklog(http_client, config, post_request_action);
2610 } else {
2611 post_request_action(error_message.empty(), error_message, rcode);
2612 }
2613 });
2614}
2615
2616 bool
2617 HAService::sendLeaseUpdatesFromBacklog() {
2618 auto num_updates = lease_update_backlog_.size();
2619 if (num_updates == 0) {
2620 LOG_INFO(ha_logger, HA_LEASES_BACKLOG_NOTHING_TO_SEND)
2621 .arg(config_->getThisServerName());
2622 return (true);
2623 }
2624
2625 IOServicePtr io_service(new IOService());
2626 HttpClient client(io_service, false);
2627 auto remote_config = config_->getFailoverPeerConfig();
2628 bool updates_successful = true;
2629
2630 LOG_INFO(ha_logger, HA_LEASES_BACKLOG_START)
2631 .arg(config_->getThisServerName())
2632 .arg(num_updates)
2633 .arg(remote_config->getName());
2634
2635 asyncSendLeaseUpdatesFromBacklog(client, remote_config,
2636 [&](const bool success, const std::string&, const int) {
2637 io_service->stop();
2638 updates_successful = success;
2639 });
2640
2641 // Measure duration of the updates.
2642 Stopwatch stopwatch;
2643
2644 // Run the IO service until it is stopped by the callback. This makes it synchronous.
2645 io_service->run();
2646
2647 // End measuring duration.
2648 stopwatch.stop();
2649
2650 client.stop();
2651
2652 io_service->stopAndPoll();
2653
2654 if (updates_successful) {
2655 LOG_INFO(ha_logger, HA_LEASES_BACKLOG_SUCCESS)
2656 .arg(config_->getThisServerName())
2657 .arg(remote_config->getName())
2658 .arg(stopwatch.logFormatLastDuration());
2659 }
2660
2661 return (updates_successful);
2662}
2663
2664 void
2665 HAService::asyncSendHAReset(HttpClient& http_client,
2666 const HAConfig::PeerConfigPtr& config,
2667 PostRequestCallback post_request_action) {
2668 ConstElementPtr command = CommandCreator::createHAReset(config_->getThisServerName(),
2669 server_type_);
2670
2671 // Create HTTP/1.1 request including our command.
2672 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2673 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2674 HostHttpHeader(config->getUrl().getStrippedHostname()));
2675 config->addBasicAuthHttpHeader(request);
2676 request->setBodyAsJson(command);
2677 request->finalize();
2678
2679 // Response object should also be created because the HTTP client needs
2680 // to know the type of the expected response.
2681 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2682
2683 http_client.asyncSendRequest(config->getUrl(), config->getTlsContext(),
2684 request, response,
2685 [this, config, post_request_action]
2686 (const boost::system::error_code& ec,
2687 const HttpResponsePtr& response,
2688 const std::string& error_str) {
2689
2690 int rcode = 0;
2691 std::string error_message;
2692
2693 if (ec || !error_str.empty()) {
2694 error_message = (ec ? ec.message() : error_str);
2695 LOG_WARN(ha_logger, HA_RESET_COMMUNICATIONS_FAILED)
2696 .arg(config_->getThisServerName())
2697 .arg(config->getLogLabel())
2698 .arg(ec ? ec.message() : error_str);
2699
2700 } else {
2701 // Handle third group of errors.
2702 try {
2703 auto args = verifyAsyncResponse(response, rcode);
2704 } catch (const std::exception& ex) {
2705 error_message = ex.what();
2706 LOG_WARN(ha_logger, HA_RESET_FAILED)
2707 .arg(config_->getThisServerName())
2708 .arg(config->getLogLabel())
2709 .arg(ex.what());
2710 }
2711 }
2712
2713 post_request_action(error_message.empty(), error_message, rcode);
2714 });
2715}
2716
2717 bool
2718 HAService::sendHAReset() {
2719 IOServicePtr io_service(new IOService());
2720 HttpClient client(io_service, false);
2721 auto remote_config = config_->getFailoverPeerConfig();
2722 bool reset_successful = true;
2723
2724 asyncSendHAReset(client, remote_config,
2725 [&](const bool success, const std::string&, const int) {
2726 io_service->stop();
2727 reset_successful = success;
2728 });
2729
2730 // Run the IO service until it is stopped by the callback. This makes it synchronous.
2731 io_service->run();
2732
2733 client.stop();
2734
2735 io_service->stopAndPoll();
2736
2737 return (reset_successful);
2738}
2739
2740 ConstElementPtr
2741 HAService::processScopes(const std::vector<std::string>& scopes) {
2742 try {
2743 query_filter_.serveScopes(scopes);
2744 adjustNetworkState();
2745
2746 } catch (const std::exception& ex) {
2747 return (createAnswer(CONTROL_RESULT_ERROR, ex.what()));
2748 }
2749
2750 return (createAnswer(CONTROL_RESULT_SUCCESS, "New HA scopes configured."));
2751}
2752
2753 ConstElementPtr
2754 HAService::processContinue() {
2755 if (unpause()) {
2756 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine continues."));
2757 }
2758 return (createAnswer(CONTROL_RESULT_SUCCESS, "HA state machine is not paused."));
2759}
2760
2761 ConstElementPtr
2762 HAService::processMaintenanceNotify(const bool cancel, const std::string& state) {
2763 if (cancel) {
2764 if (getCurrState() != HA_IN_MAINTENANCE_ST) {
2765 return (createAnswer(CONTROL_RESULT_ERROR, "Unable to cancel the"
2766 " maintenance for the server not in the"
2767 " in-maintenance state."));
2768 }
2769
2770 try {
2771 communication_state_->setPartnerState(state);
2772
2773 } catch (...) {
2774 // Hopefully the received state is correct. If it isn't, let's set the
2775 // partner state to unavailable and count on the state machine to resolve.
2776 communication_state_->setPartnerUnavailable();
2777 }
2778 postNextEvent(HA_MAINTENANCE_CANCEL_EVT);
2779 // In rare cases the previous state may be the server's current state. Transitioning
2780 // to it would cause a deadlock and the server will remain stuck in maintenance.
2781 // In these cases let's simply transition to the waiting state and the state machine
2782 // should solve it.
2783 verboseTransition(getPrevState() == HA_IN_MAINTENANCE_ST ? HA_WAITING_ST : getPrevState());
2784 runModel(NOP_EVT);
2785
2786 // Communicate the new state to the partner.
2787 ElementPtr arguments = Element::createMap();
2788 std::string state_label = getState(getCurrState())->getLabel();
2789 arguments->set("state", Element::create(state_label));
2790
2791 return (createAnswer(CONTROL_RESULT_SUCCESS, "Server maintenance canceled.", arguments));
2792 }
2793
2794 switch (getCurrState()) {
2795 case HA_BACKUP_ST:
2796 case HA_PARTNER_IN_MAINTENANCE_ST:
2797 case HA_TERMINATED_ST:
2798 // The reason why we don't return an error result here is that we have to
2799 // have a way to distinguish between the errors caused by the communication
2800 // issues and the cases when there is no communication error but the server
2801 // is not allowed to enter the in-maintenance state. In the former case, the
2802 // partner would go to partner-down. In the case signaled by the special
2803 // result code entering the maintenance state is not allowed.
2804 return (createAnswer(HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED,
2805 "Unable to transition the server from the "
2806 + stateToString(getCurrState()) + " to"
2807 " in-maintenance state."));
2808 default:
2809 verboseTransition(HA_IN_MAINTENANCE_ST);
2810 runModel(HA_MAINTENANCE_NOTIFY_EVT);
2811 }
2812 return (createAnswer(CONTROL_RESULT_SUCCESS, "Server is in-maintenance state."));
2813}
2814
2815 ConstElementPtr
2816 HAService::processMaintenanceStart() {
2817 switch (getCurrState()) {
2818 case HA_BACKUP_ST:
2819 case HA_IN_MAINTENANCE_ST:
2820 case HA_PARTNER_IN_MAINTENANCE_ST:
2821 case HA_TERMINATED_ST:
2822 return (createAnswer(CONTROL_RESULT_ERROR, "Unable to transition the server from"
2823 " the " + stateToString(getCurrState()) + " to"
2824 " partner-in-maintenance state."));
2825 default:
2826 ;
2827 }
2828
2829 HAConfig::PeerConfigPtr remote_config = config_->getFailoverPeerConfig();
2830
2831 // Create HTTP/1.1 request including ha-maintenance-notify command
2832 // with the cancel flag set to false.
2833 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2834 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2835 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2836 remote_config->addBasicAuthHttpHeader(request);
2837 request->setBodyAsJson(CommandCreator::createMaintenanceNotify(config_->getThisServerName(),
2838 false, getCurrState(), server_type_));
2839 request->finalize();
2840
2841 // Response object should also be created because the HTTP client needs
2842 // to know the type of the expected response.
2843 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2844
2845 IOServicePtr io_service(new IOService());
2846 HttpClient client(io_service, false);
2847
2848 boost::system::error_code captured_ec;
2849 std::string captured_error_message;
2850 int captured_rcode = 0;
2851
2852 // Schedule asynchronous HTTP request.
2853 client.asyncSendRequest(remote_config->getUrl(),
2854 remote_config->getTlsContext(),
2855 request, response,
2856 [this, remote_config, &io_service, &captured_ec, &captured_error_message,
2857 &captured_rcode]
2858 (const boost::system::error_code& ec,
2859 const HttpResponsePtr& response,
2860 const std::string& error_str) {
2861
2862 io_service->stop();
2863
2864 // There are three possible groups of errors. One is the IO error
2865 // causing issues in communication with the peer. Another one is
2866 // an HTTP parsing error. The last type of error is when non-success
2867 // error code is returned in the response carried in the HTTP message
2868 // or if the JSON response is otherwise broken.
2869
2870 std::string error_message;
2871
2872 // Handle first two groups of errors.
2873 if (ec || !error_str.empty()) {
2874 error_message = (ec ? ec.message() : error_str);
2875 LOG_ERROR(ha_logger, HA_MAINTENANCE_NOTIFY_COMMUNICATIONS_FAILED)
2876 .arg(config_->getThisServerName())
2877 .arg(remote_config->getLogLabel())
2878 .arg(error_message);
2879
2880 } else {
2881
2882 // Handle third group of errors.
2883 try {
2884 static_cast<void>(verifyAsyncResponse(response, captured_rcode));
2885
2886 } catch (const std::exception& ex) {
2887 error_message = ex.what();
2888 LOG_ERROR(ha_logger, HA_MAINTENANCE_NOTIFY_FAILED)
2889 .arg(config_->getThisServerName())
2890 .arg(remote_config->getLogLabel())
2891 .arg(error_message);
2892 }
2893 }
2894
2895 // If there was an error communicating with the partner, mark the
2896 // partner as unavailable.
2897 if (!error_message.empty()) {
2898 communication_state_->setPartnerUnavailable();
2899 }
2900
2901 captured_ec = ec;
2902 captured_error_message = error_message;
2903 },
2904 HttpClient::RequestTimeout(TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST),
2905 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
2906 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
2907 std::bind(&HAService::clientCloseHandler, this, ph::_1)
2908 );
2909
2910 // Run the IO service until it is stopped by any of the callbacks. This
2911 // makes it synchronous.
2912 io_service->run();
2913
2914 client.stop();
2915
2916 io_service->stopAndPoll();
2917
2918 // If there was a communication problem with the partner we assume that
2919 // the partner is already down while we receive this command.
2920 if (captured_ec || (captured_rcode == CONTROL_RESULT_ERROR)) {
2921 postNextEvent(HA_MAINTENANCE_START_EVT);
2922 verboseTransition(HA_PARTNER_DOWN_ST);
2923 runModel(NOP_EVT);
2924 return (createAnswer(CONTROL_RESULT_SUCCESS,
2925 "Server is now in the partner-down state as its"
2926 " partner appears to be offline for maintenance."));
2927
2928 } else if (captured_rcode == CONTROL_RESULT_SUCCESS) {
2929 // If the partner responded indicating no error it means that the
2930 // partner has been transitioned to the in-maintenance state. In that
2931 // case we transition to the partner-in-maintenance state.
2932 postNextEvent(HA_MAINTENANCE_START_EVT);
2933 verboseTransition(HA_PARTNER_IN_MAINTENANCE_ST);
2934 runModel(NOP_EVT);
2935
2936 } else {
2937 // Partner server returned a special status code which means that it can't
2938 // transition to the partner-in-maintenance state.
2939 return (createAnswer(CONTROL_RESULT_ERROR, "Unable to transition to the"
2940 " partner-in-maintenance state. The partner server responded"
2941 " with the following message to the ha-maintenance-notify"
2942 " command: " + captured_error_message + "."));
2943
2944 }
2945
2946 return (createAnswer(CONTROL_RESULT_SUCCESS,
2947 "Server is now in the partner-in-maintenance state"
2948 " and its partner is in-maintenance state. The partner"
2949 " can be now safely shut down."));
2950}
2951
2952 ConstElementPtr
2953 HAService::processMaintenanceCancel() {
2954 if (getCurrState() != HA_PARTNER_IN_MAINTENANCE_ST) {
2955 return (createAnswer(CONTROL_RESULT_ERROR, "Unable to cancel maintenance"
2956 " request because the server is not in the"
2957 " partner-in-maintenance state."));
2958 }
2959
2960 // This is the state the server will transition to if the notification to the
2961 // partner is successful.
2962 int next_state = getPrevState() == HA_PARTNER_IN_MAINTENANCE_ST ? HA_WAITING_ST : getPrevState();
2963
2964 HAConfig::PeerConfigPtr remote_config = config_->getFailoverPeerConfig();
2965
2966 // Create HTTP/1.1 request including ha-maintenance-notify command
2967 // with the cancel flag set to true.
2968 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
2969 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
2970 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
2971 remote_config->addBasicAuthHttpHeader(request);
2972 request->setBodyAsJson(CommandCreator::createMaintenanceNotify(config_->getThisServerName(),
2973 true,
2974 next_state,
2975 server_type_));
2976 request->finalize();
2977
2978 // Response object should also be created because the HTTP client needs
2979 // to know the type of the expected response.
2980 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
2981
2982 IOServicePtr io_service(new IOService());
2983 HttpClient client(io_service, false);
2984
2985 std::string error_message;
2986
2987 // Schedule asynchronous HTTP request.
2988 client.asyncSendRequest(remote_config->getUrl(),
2989 remote_config->getTlsContext(),
2990 request, response,
2991 [this, remote_config, &io_service, &error_message]
2992 (const boost::system::error_code& ec,
2993 const HttpResponsePtr& response,
2994 const std::string& error_str) {
2995
2996 io_service->stop();
2997
2998 // Handle first two groups of errors.
2999 if (ec || !error_str.empty()) {
3000 error_message = (ec ? ec.message() : error_str);
3001 LOG_ERROR(ha_logger, HA_MAINTENANCE_NOTIFY_CANCEL_COMMUNICATIONS_FAILED)
3002 .arg(config_->getThisServerName())
3003 .arg(remote_config->getLogLabel())
3004 .arg(error_message);
3005
3006 } else {
3007
3008 // Handle third group of errors.
3009 try {
3010 int rcode = 0;
3011 ConstElementPtr args = verifyAsyncResponse(response, rcode);
3012
3013 // Partner's state has changed after the notification. However, we don't know
3014 // its new state. We'll check if the partner returned its state. If it didn't,
3015 // we set the unavailable state as a default.
3016 communication_state_->setPartnerUnavailable();
3017
3018 // Newer Kea versions return the state of the notified server.
3019 // Older versions don't, so the arguments may not be present.
3020 if (args && args->getType() == Element::map) {
3021 // Arguments may include partner's state.
3022 ConstElementPtr state = args->get("state");
3023 if (state) {
3024 if (state->getType() != Element::string) {
3025 isc_throw(CtrlChannelError, "server state not returned in response"
3026 " to a ha-heartbeat command or it is not a string");
3027 }
3028 communication_state_->setPartnerState(state->stringValue());
3029 }
3030 }
3031 } catch (const std::exception& ex) {
3032 error_message = ex.what();
3033 LOG_ERROR(ha_logger, HA_MAINTENANCE_NOTIFY_CANCEL_FAILED)
3034 .arg(config_->getThisServerName())
3035 .arg(remote_config->getLogLabel())
3036 .arg(error_message);
3037 }
3038 }
3039
3040 // If there was an error communicating with the partner, mark the
3041 // partner as unavailable.
3042 if (!error_message.empty()) {
3043 communication_state_->setPartnerUnavailable();
3044 }
3045 },
3046 HttpClient::RequestTimeout(TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST),
3047 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
3048 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
3049 std::bind(&HAService::clientCloseHandler, this, ph::_1)
3050 );
3051
3052 // Run the IO service until it is stopped by any of the callbacks. This
3053 // makes it synchronous.
3054 io_service->run();
3055
3056 client.stop();
3057
3058 io_service->stopAndPoll();
3059
3060 // There was an error in communication with the partner or the
3061 // partner was unable to revert its state.
3062 if (!error_message.empty()) {
3063 return (createAnswer(CONTROL_RESULT_ERROR,
3064 "Unable to cancel maintenance. The partner server responded"
3065 " with the following message to the ha-maintenance-notify"
3066 " command: " + error_message + "."));
3067 }
3068
3069 // Successfully reverted partner's state. Let's also revert our state to the
3070 // previous one. Avoid returning to the partner-in-maintenance if it was
3071 // the previous state.
3072 postNextEvent(HA_MAINTENANCE_CANCEL_EVT);
3073 verboseTransition(next_state);
3074 runModel(NOP_EVT);
3075
3076 return (createAnswer(CONTROL_RESULT_SUCCESS,
3077 "Server maintenance successfully canceled."));
3078}
3079
3080 void
3081 HAService::asyncSyncCompleteNotify(HttpClient& http_client,
3082 const HAConfig::PeerConfigPtr& remote_config,
3083 PostRequestCallback post_request_action) {
3084 // Create HTTP/1.1 request including our command.
3085 PostHttpRequestJsonPtr request = boost::make_shared<PostHttpRequestJson>
3086 (HttpRequest::Method::HTTP_POST, "/", HttpVersion::HTTP_11(),
3087 HostHttpHeader(remote_config->getUrl().getStrippedHostname()));
3088
3089 remote_config->addBasicAuthHttpHeader(request);
3090 request->setBodyAsJson(CommandCreator::createSyncCompleteNotify(getRemoteOrigin(),
3091 config_->getThisServerName(),
3092 server_type_));
3093 request->finalize();
3094
3095 // Response object should also be created because the HTTP client needs
3096 // to know the type of the expected response.
3097 HttpResponseJsonPtr response = boost::make_shared<HttpResponseJson>();
3098
3099 // Schedule asynchronous HTTP request.
3100 http_client.asyncSendRequest(remote_config->getUrl(),
3101 remote_config->getTlsContext(),
3102 request, response,
3103 [this, remote_config, post_request_action]
3104 (const boost::system::error_code& ec,
3105 const HttpResponsePtr& response,
3106 const std::string& error_str) {
3107
3108 // There are three possible groups of errors. One is the IO error
3109 // causing issues in communication with the peer. Another one is an
3110 // HTTP parsing error. The last type of error is when non-success
3111 // error code is returned in the response carried in the HTTP message
3112 // or if the JSON response is otherwise broken.
3113
3114 int rcode = 0;
3115 std::string error_message;
3116
3117 // Handle first two groups of errors.
3118 if (ec || !error_str.empty()) {
3119 error_message = (ec ? ec.message() : error_str);
3120 LOG_ERROR(ha_logger, HA_SYNC_COMPLETE_NOTIFY_COMMUNICATIONS_FAILED)
3121 .arg(config_->getThisServerName())
3122 .arg(remote_config->getLogLabel())
3123 .arg(error_message);
3124
3125 } else {
3126
3127 // Handle third group of errors.
3128 try {
3129 static_cast<void>(verifyAsyncResponse(response, rcode));
3130
3131 } catch (const CommandUnsupportedError& ex) {
3132 rcode = CONTROL_RESULT_COMMAND_UNSUPPORTED;
3133
3134 } catch (const std::exception& ex) {
3135 error_message = ex.what();
3136 LOG_ERROR(ha_logger, HA_SYNC_COMPLETE_NOTIFY_FAILED)
3137 .arg(config_->getThisServerName())
3138 .arg(remote_config->getLogLabel())
3139 .arg(error_message);
3140 }
3141 }
3142
3143 // If there was an error communicating with the partner, mark the
3144 // partner as unavailable.
3145 if (!error_message.empty()) {
3146 communication_state_->setPartnerUnavailable();
3147 }
3148
3149 // Invoke post request action if it was specified.
3150 if (post_request_action) {
3151 post_request_action(error_message.empty(),
3152 error_message,
3153 rcode);
3154 }
3155 },
3156 HttpClient::RequestTimeout(TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST),
3157 std::bind(&HAService::clientConnectHandler, this, ph::_1, ph::_2),
3158 std::bind(&HAService::clientHandshakeHandler, this, ph::_1),
3159 std::bind(&HAService::clientCloseHandler, this, ph::_1)
3160 );
3161}
3162
3163 ConstElementPtr
3164 HAService::processSyncCompleteNotify(const unsigned int origin_id) {
3165 if (getCurrState() == HA_PARTNER_DOWN_ST) {
3166 sync_complete_notified_ = true;
3167 // We're in the partner-down state and the partner notified us
3168 // that it has synchronized its database. We can't enable the
3169 // service yet, because it may result in some new lease allocations
3170 // that the partner would miss (we don't send lease updates in the
3171 // partner-down state). We must first send the heartbeat and let
3172 // the state machine resolve the situation between the partners.
3173 // It may unblock the network service.
3174 network_state_->disableService(getLocalOrigin());
3175 }
3176 // Release the network state lock for the remote origin because we have
3177 // acquired the local network state lock above (partner-down state), or
3178 // we don't need the lock (other states).
3179 network_state_->enableService(origin_id);
3180 return (createAnswer(CONTROL_RESULT_SUCCESS,
3181 "Server successfully notified about the synchronization completion."));
3182}
3183
3184 ConstElementPtr
3185 HAService::verifyAsyncResponse(const HttpResponsePtr& response, int& rcode) {
3186 // Set the return code to error in case of early throw.
3187 rcode = CONTROL_RESULT_ERROR;
3188 // The response must cast to JSON type.
3189 HttpResponseJsonPtr json_response =
3190 boost::dynamic_pointer_cast<HttpResponseJson>(response);
3191 if (!json_response) {
3192 isc_throw(CtrlChannelError, "no valid HTTP response found");
3193 }
3194
3195 // Body holds the response to our command.
3196 ConstElementPtr body = json_response->getBodyAsJson();
3197 if (!body) {
3198 isc_throw(CtrlChannelError, "no body found in the response");
3199 }
3200
3201 // Body should contain a list of responses from multiple servers.
3202 if (body->getType() != Element::list) {
3203 // Some control agent errors are returned as a map.
3204 if (body->getType() == Element::map) {
3205 ElementPtr list = Element::createList();
3206 ElementPtr answer = Element::createMap();
3207 answer->set(CONTROL_RESULT, Element::create(rcode));
3208 ConstElementPtr text = body->get(CONTROL_TEXT);
3209 if (text) {
3210 answer->set(CONTROL_TEXT, text);
3211 }
3212 list->add(answer);
3213 body = list;
3214 } else {
3215 isc_throw(CtrlChannelError, "body of the response must be a list");
3216 }
3217 }
3218
3219 // There must be at least one response.
3220 if (body->empty()) {
3221 isc_throw(CtrlChannelError, "list of responses must not be empty");
3222 }
3223
3224 // Check if the status code of the first response. We don't support multiple
3225 // at this time, because we always send a request to a single location.
3226 ConstElementPtr args = parseAnswer(rcode, body->get(0));
3227 if (rcode == CONTROL_RESULT_SUCCESS) {
3228 return (args);
3229 }
3230
3231 std::ostringstream s;
3232
3233 // The empty status can occur for the lease6-bulk-apply command. In that
3234 // case, the response may contain conflicted or erred leases within the
3235 // arguments, rather than globally. For other error cases let's construct
3236 // the error message from the global values.
3237 if (rcode != CONTROL_RESULT_EMPTY) {
3238 // Include an error text if available.
3239 if (args && args->getType() == Element::string) {
3240 s << args->stringValue() << " (";
3241 }
3242 // Include an error code.
3243 s << "error code " << rcode << ")";
3244 }
3245
3246 switch (rcode) {
3247 case CONTROL_RESULT_COMMAND_UNSUPPORTED:
3248 isc_throw(CommandUnsupportedError, s.str());
3249
3250 case CONTROL_RESULT_CONFLICT:
3251 isc_throw(ConflictError, s.str());
3252
3253 case CONTROL_RESULT_EMPTY:
3254 // Handle the lease6-bulk-apply error cases.
3255 if (args && (args->getType() == Element::map)) {
3256 auto failed_leases = args->get("failed-leases");
3257 if (!failed_leases || (failed_leases->getType() != Element::list)) {
3258 // If there are no failed leases there is nothing to do.
3259 break;
3260 }
3261 auto conflict = false;
3262 ConstElementPtr conflict_error_message;
3263 for (unsigned i = 0; i < failed_leases->size(); ++i) {
3264 auto lease = failed_leases->get(i);
3265 if (!lease || lease->getType() != Element::map) {
3266 continue;
3267 }
3268 auto result = lease->get("result");
3269 if (!result || result->getType() != Element::integer) {
3270 continue;
3271 }
3272 auto error_message = lease->get("error-message");
3273 // Error status code takes precedence over the conflict.
3274 if (result->intValue() == CONTROL_RESULT_ERROR) {
3275 if (error_message && error_message->getType()) {
3276 s << error_message->stringValue() << " (";
3277 }
3278 s << "error code " << result->intValue() << ")";
3279 isc_throw(CtrlChannelError, s.str());
3280 }
3281 if (result->intValue() == CONTROL_RESULT_CONFLICT) {
3282 // Let's record the conflict but there may still be some
3283 // leases with an error status code, so do not throw the
3284 // conflict exception yet.
3285 conflict = true;
3286 conflict_error_message = error_message;
3287 }
3288 }
3289 if (conflict) {
3290 // There are no errors. There are only conflicts. Throw
3291 // appropriate exception.
3292 if (conflict_error_message &&
3293 (conflict_error_message->getType() == Element::string)) {
3294 s << conflict_error_message->stringValue() << " (";
3295 }
3296 s << "error code " << CONTROL_RESULT_CONFLICT << ")";
3297 isc_throw(ConflictError, s.str());
3298 }
3299 }
3300 break;
3301 default:
3302 isc_throw(CtrlChannelError, s.str());
3303 }
3304 return (args);
3305}
3306
3307 bool
3308 HAService::clientConnectHandler(const boost::system::error_code& ec, int tcp_native_fd) {
3309
3310 // If client is running it's own IOService we do NOT want to
3311 // register the socket with IfaceMgr.
3312 if (client_->getThreadIOService()) {
3313 return (true);
3314 }
3315
3316 // If things look OK register the socket with Interface Manager. Note
3317 // we don't register if the FD is < 0 to avoid an exception throw.
3318 // It is unlikely that this will occur but we want to be liberal
3319 // and avoid issues.
3320 if ((!ec || (ec.value() == boost::asio::error::in_progress))
3321 && (tcp_native_fd >= 0)) {
3322 // External socket callback is a NOP. Ready events handlers are
3323 // run by an explicit call IOService ready in kea-dhcp<n> code.
3324 // We are registering the socket only to interrupt main-thread
3325 // select().
3326 IfaceMgr::instance().addExternalSocket(tcp_native_fd,
3327 std::bind(&HAService::socketReadyHandler, this, ph::_1)
3328 );
3329 }
3330
3331 // If ec.value() == boost::asio::error::already_connected, we should already
3332 // be registered, so nothing to do. If it is any other value, then connect
3333 // failed and Connection logic should handle that, not us, so no matter
3334 // what happens we're returning true.
3335 return (true);
3336}
3337
3338 void
3339 HAService::socketReadyHandler(int tcp_native_fd) {
3340 // If the socket is ready but does not belong to one of our client's
3341 // ongoing transactions, we close it. This will unregister it from
3342 // IfaceMgr and ensure the client starts over with a fresh connection
3343 // if it needs to do so.
3344 client_->closeIfOutOfBand(tcp_native_fd);
3345}
3346
3347 void
3348 HAService::clientCloseHandler(int tcp_native_fd) {
3349 if (tcp_native_fd >= 0) {
3350 IfaceMgr::instance().deleteExternalSocket(tcp_native_fd);
3351 }
3352}
3353
3354 size_t
3355 HAService::pendingRequestSize() {
3356 if (MultiThreadingMgr::instance().getMode()) {
3357 std::lock_guard<std::mutex> lock(mutex_);
3358 return (pending_requests_.size());
3359 } else {
3360 return (pending_requests_.size());
3361 }
3362}
3363
3364 template<typename QueryPtrType>
3365 int
3366 HAService::getPendingRequest(const QueryPtrType& query) {
3367 if (MultiThreadingMgr::instance().getMode()) {
3368 std::lock_guard<std::mutex> lock(mutex_);
3369 return (getPendingRequestInternal(query));
3370 } else {
3371 return (getPendingRequestInternal(query));
3372 }
3373}
3374
3375 template<typename QueryPtrType>
3376 int
3377HAService::getPendingRequestInternal(const QueryPtrType& query) {
3378 if (pending_requests_.count(query) == 0) {
3379 return (0);
3380 } else {
3381 return (pending_requests_[query]);
3382 }
3383}
3384
3385 void
3386 HAService::checkPermissionsClientAndListener() {
3387 // Since this function is used as CS callback all exceptions must be
3388 // suppressed (except the @ref MultiThreadingInvalidOperation), unlikely
3389 // though they may be.
3390 // The @ref MultiThreadingInvalidOperation is propagated to the scope of the
3391 // @ref MultiThreadingCriticalSection constructor.
3392 try {
3393 if (client_) {
3394 client_->checkPermissions();
3395 }
3396
3397 if (listener_) {
3398 listener_->checkPermissions();
3399 }
3400 } catch (const isc::MultiThreadingInvalidOperation& ex) {
3401 LOG_ERROR(ha_logger, HA_PAUSE_CLIENT_LISTENER_ILLEGAL)
3402 .arg(config_->getThisServerName())
3403 .arg(ex.what());
3404 // The exception needs to be propagated to the caller of the
3405 // @ref MultiThreadingCriticalSection constructor.
3406 throw;
3407 } catch (const std::exception& ex) {
3408 LOG_ERROR(ha_logger, HA_PAUSE_CLIENT_LISTENER_FAILED)
3409 .arg(config_->getThisServerName())
3410 .arg(ex.what());
3411 }
3412}
3413
3414 void
3415 HAService::startClientAndListener() {
3416 // Add critical section callbacks.
3417 MultiThreadingMgr::instance().addCriticalSectionCallbacks(getCSCallbacksSetName(),
3418 std::bind(&HAService::checkPermissionsClientAndListener, this),
3419 std::bind(&HAService::pauseClientAndListener, this),
3420 std::bind(&HAService::resumeClientAndListener, this));
3421
3422 if (client_) {
3423 client_->start();
3424 }
3425
3426 if (listener_) {
3427 listener_->start();
3428 }
3429}
3430
3431 void
3432 HAService::pauseClientAndListener() {
3433 // Since this function is used as CS callback all exceptions must be
3434 // suppressed, unlikely though they may be.
3435 try {
3436 if (client_) {
3437 client_->pause();
3438 }
3439
3440 if (listener_) {
3441 listener_->pause();
3442 }
3443 } catch (const std::exception& ex) {
3444 LOG_ERROR(ha_logger, HA_PAUSE_CLIENT_LISTENER_FAILED)
3445 .arg(ex.what());
3446 }
3447}
3448
3449 void
3450 HAService::resumeClientAndListener() {
3451 // Since this function is used as CS callback all exceptions must be
3452 // suppressed, unlikely though they may be.
3453 try {
3454 if (client_) {
3455 client_->resume();
3456 }
3457
3458 if (listener_) {
3459 listener_->resume();
3460 }
3461 } catch (std::exception& ex) {
3462 LOG_ERROR(ha_logger, HA_RESUME_CLIENT_LISTENER_FAILED)
3463 .arg(config_->getThisServerName())
3464 .arg(ex.what());
3465 }
3466}
3467
3468 void
3469 HAService::stopClientAndListener() {
3470 // Remove critical section callbacks.
3471 MultiThreadingMgr::instance().removeCriticalSectionCallbacks(getCSCallbacksSetName());
3472
3473 if (client_) {
3474 client_->stop();
3475 }
3476
3477 if (listener_) {
3478 listener_->stop();
3479 }
3480}
3481
3482// Explicit instantiations.
3483 template int HAService::getPendingRequest(const Pkt4Ptr&);
3484 template int HAService::getPendingRequest(const Pkt6Ptr&);
3485
3486} // end of namespace isc::ha
3487} // end of namespace isc
if
if(!(yy_init))
Definition agent_lexer.cc:1900
boost_time_utils.h
cfgmgr.h
Element::create
static ElementPtr create(const Position &pos=ZERO_POSITION())
Definition data.cc:249
Element::map
@ map
Definition data.h:147
Element::integer
@ integer
Definition data.h:140
Element::list
@ list
Definition data.h:146
Element::string
@ string
Definition data.h:144
Element::createMap
static ElementPtr createMap(const Position &pos=ZERO_POSITION())
Creates an empty MapElement type ElementPtr.
Definition data.cc:304
Element::createList
static ElementPtr createList(const Position &pos=ZERO_POSITION())
Creates an empty ListElement type ElementPtr.
Definition data.cc:299
isc::Exception::what
virtual const char * what() const
Returns a C-style character string of the cause of the exception.
Definition exceptions/exceptions.cc:32
isc::MultiThreadingInvalidOperation
Exception thrown when a worker thread is trying to stop or pause the respective thread pool (which wo...
Definition exceptions/exceptions.h:183
isc::Unexpected
A generic exception that is thrown when an unexpected error condition occurs.
Definition exceptions/exceptions.h:153
isc::asiolink::IOAddress
The IOAddress class represents an IP addresses (version agnostic).
Definition io_address.h:45
isc::asiolink::IOAddress::IPV4_ZERO_ADDRESS
static const IOAddress & IPV4_ZERO_ADDRESS()
Returns an address set to all zeros.
Definition io_address.h:277
isc::asiolink::IOService
The IOService class is a wrapper for the ASIO io_context class.
Definition io_service.h:32
isc::config::CmdHttpListener
A multi-threaded HTTP listener that can process API commands requests.
Definition cmd_http_listener.h:32
isc::config::CmdResponseCreator::command_accept_list_
static std::unordered_set< std::string > command_accept_list_
The server command accept list.
Definition cmd_response_creator.h:94
isc::config::CtrlChannelError
A standard control channel exception that is thrown if a function is there is a problem with one of t...
Definition command_interpreter.h:63
isc::dhcp::IfaceMgr::deleteExternalSocket
void deleteExternalSocket(int socketfd)
Deletes external socket.
Definition iface_mgr.cc:352
isc::dhcp::IfaceMgr::instance
static IfaceMgr & instance()
IfaceMgr is a singleton class.
Definition iface_mgr.cc:54
isc::dhcp::IfaceMgr::addExternalSocket
void addExternalSocket(int socketfd, SocketCallback callback)
Adds external socket and a callback.
Definition iface_mgr.cc:329
isc::ha::CommandCreator::createLease4Delete
static data::ConstElementPtr createLease4Delete(const dhcp::Lease4 &lease4)
Creates lease4-del command.
Definition command_creator.cc:104
isc::ha::CommandCreator::createHeartbeat
static data::ConstElementPtr createHeartbeat(const std::string &server_name, const HAServerType &server_type)
Creates ha-heartbeat command for DHCP server.
Definition command_creator.cc:83
isc::ha::CommandCreator::ha_commands4_
static std::unordered_set< std::string > ha_commands4_
List of commands used by the High Availability in v4.
Definition command_creator.h:202
isc::ha::CommandCreator::createLease4Update
static data::ConstElementPtr createLease4Update(const dhcp::Lease4 &lease4)
Creates lease4-update command.
Definition command_creator.cc:93
isc::ha::CommandCreator::createSyncCompleteNotify
static data::ConstElementPtr createSyncCompleteNotify(const unsigned int origin_id, const std::string &server_name, const HAServerType &server_type)
Creates ha-sync-complete-notify command.
Definition command_creator.cc:275
isc::ha::CommandCreator::createLease6BulkApply
static data::ConstElementPtr createLease6BulkApply(const dhcp::Lease6CollectionPtr &leases, const dhcp::Lease6CollectionPtr &deleted_leases)
Creates lease6-bulk-apply command.
Definition command_creator.cc:146
isc::ha::CommandCreator::createLease6GetPage
static data::ConstElementPtr createLease6GetPage(const dhcp::Lease6Ptr &lease6, const uint32_t limit)
Creates lease6-get-page command.
Definition command_creator.cc:236
isc::ha::CommandCreator::createDHCPDisable
static data::ConstElementPtr createDHCPDisable(const unsigned int origin_id, const unsigned int max_period, const HAServerType &server_type)
Creates dhcp-disable command for DHCP server.
Definition command_creator.cc:42
isc::ha::CommandCreator::createDHCPEnable
static data::ConstElementPtr createDHCPEnable(const unsigned int origin_id, const HAServerType &server_type)
Creates dhcp-enable command for DHCP server.
Definition command_creator.cc:60
isc::ha::CommandCreator::createMaintenanceNotify
static data::ConstElementPtr createMaintenanceNotify(const std::string &server_name, const bool cancel, const int state, const HAServerType &server_type)
Creates ha-maintenance-notify command.
Definition command_creator.cc:261
isc::ha::CommandCreator::ha_commands6_
static std::unordered_set< std::string > ha_commands6_
List of commands used by the High Availability in v6.
Definition command_creator.h:205
isc::ha::CommandCreator::createHAReset
static data::ConstElementPtr createHAReset(const std::string &server_name, const HAServerType &server_type)
Creates ha-reset command.
Definition command_creator.cc:73
isc::ha::CommandCreator::createLease4GetPage
static data::ConstElementPtr createLease4GetPage(const dhcp::Lease4Ptr &lease4, const uint32_t limit)
Creates lease4-get-page command.
Definition command_creator.cc:121
isc::ha::CommunicationState4
Holds communication state between DHCPv4 servers.
Definition communication_state.h:788
isc::ha::CommunicationState6
Holds communication state between DHCPv6 servers.
Definition communication_state.h:993
isc::ha::HAConfig::PeerConfig::Role
Role
Server's role in the High Availability setup.
Definition ha_config.h:83
isc::ha::HAConfig::PeerConfig::STANDBY
@ STANDBY
Definition ha_config.h:86
isc::ha::HAConfig::PeerConfig::PRIMARY
@ PRIMARY
Definition ha_config.h:84
isc::ha::HAConfig::PeerConfig::BACKUP
@ BACKUP
Definition ha_config.h:87
isc::ha::HAConfig::PeerConfig::roleToString
static std::string roleToString(const HAConfig::PeerConfig::Role &role)
Returns role name.
Definition ha_config.cc:82
isc::ha::HAConfig::HOT_STANDBY
@ HOT_STANDBY
Definition ha_config.h:57
isc::ha::HAConfig::PASSIVE_BACKUP
@ PASSIVE_BACKUP
Definition ha_config.h:58
isc::ha::HAConfig::LOAD_BALANCING
@ LOAD_BALANCING
Definition ha_config.h:56
isc::ha::HAConfig::PeerConfigMap
std::map< std::string, PeerConfigPtr > PeerConfigMap
Map of the servers' configurations.
Definition ha_config.h:245
isc::ha::HAConfig::HAModeToString
static std::string HAModeToString(const HAMode &ha_mode)
Returns HA mode name.
Definition ha_config.cc:233
isc::ha::HAConfig::PeerConfigPtr
boost::shared_ptr< PeerConfig > PeerConfigPtr
Pointer to the server's configuration.
Definition ha_config.h:242
isc::ha::HAService::HA_MAINTENANCE_START_EVT
static const int HA_MAINTENANCE_START_EVT
ha-maintenance-start command received.
Definition ha_service.h:71
isc::ha::HAService::inScope
bool inScope(dhcp::Pkt4Ptr &query4)
Checks if the DHCPv4 query should be processed by this server.
Definition ha_service.cc:1059
isc::ha::HAService::adjustNetworkState
void adjustNetworkState()
Enables or disables network state depending on the served scopes.
Definition ha_service.cc:1105
isc::ha::HAService::stopClientAndListener
void stopClientAndListener()
Stop the client and(or) listener instances.
Definition ha_service.cc:3469
isc::ha::HAService::getNormalState
int getNormalState() const
Returns normal operation state for the current configuration.
Definition ha_service.cc:1010
isc::ha::HAService::shouldQueueLeaseUpdates
bool shouldQueueLeaseUpdates(const HAConfig::PeerConfigPtr &peer_config) const
Checks if the lease updates should be queued.
Definition ha_service.cc:1624
isc::ha::HAService::HA_HEARTBEAT_COMPLETE_EVT
static const int HA_HEARTBEAT_COMPLETE_EVT
Finished heartbeat command.
Definition ha_service.h:56
isc::ha::HAService::asyncSendHAReset
void asyncSendHAReset(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, PostRequestCallback post_request_action)
Sends ha-reset command to partner asynchronously.
Definition ha_service.cc:2665
isc::ha::HAService::clientConnectHandler
bool clientConnectHandler(const boost::system::error_code &ec, int tcp_native_fd)
HttpClient connect callback handler.
Definition ha_service.cc:3308
isc::ha::HAService::asyncSyncLeases
void asyncSyncLeases()
Asynchronously reads leases from a peer and updates local lease database.
Definition ha_service.cc:2121
isc::ha::HAService::isMaintenanceCanceled
bool isMaintenanceCanceled() const
Convenience method checking if the current state is a result of canceling the maintenance.
Definition ha_service.cc:1180
isc::ha::HAService::processMaintenanceCancel
data::ConstElementPtr processMaintenanceCancel()
Processes ha-maintenance-cancel command and returns a response.
Definition ha_service.cc:2953
isc::ha::HAService::checkPermissionsClientAndListener
void checkPermissionsClientAndListener()
Check client and(or) listener current thread permissions to perform thread pool state transition.
Definition ha_service.cc:3386
isc::ha::HAService::shouldReclaim
bool shouldReclaim(const dhcp::Lease4Ptr &lease4) const
Checks if the lease should be reclaimed by this server.
Definition ha_service.cc:1089
isc::ha::HAService::asyncSendLeaseUpdate
void asyncSendLeaseUpdate(const QueryPtrType &query, const HAConfig::PeerConfigPtr &config, const data::ConstElementPtr &command, const hooks::ParkingLotHandlePtr &parking_lot)
Asynchronously sends lease update to the peer.
Definition ha_service.cc:1434
isc::ha::HAService::verboseTransition
void verboseTransition(const unsigned state)
Transitions to a desired state and logs it.
Definition ha_service.cc:939
isc::ha::HAService::sendLeaseUpdatesFromBacklog
bool sendLeaseUpdatesFromBacklog()
Attempts to send all lease updates from the backlog synchronously.
Definition ha_service.cc:2617
isc::ha::HAService::listener_
config::CmdHttpListenerPtr listener_
HTTP listener instance used to receive and respond to HA commands and lease updates.
Definition ha_service.h:1290
isc::ha::HAService::clientCloseHandler
void clientCloseHandler(int tcp_native_fd)
HttpClient close callback handler.
Definition ha_service.cc:3348
isc::ha::HAService::leaseUpdateComplete
bool leaseUpdateComplete(QueryPtrType &query, const hooks::ParkingLotHandlePtr &parking_lot)
Handle last pending request for this query.
Definition ha_service.cc:1378
isc::ha::HAService::config_
HAConfigPtr config_
Pointer to the HA hooks library configuration.
Definition ha_service.h:1280
isc::ha::HAService::processMaintenanceStart
data::ConstElementPtr processMaintenanceStart()
Processes ha-maintenance-start command and returns a response.
Definition ha_service.cc:2816
isc::ha::HAService::id_
unsigned int id_
Unique service id.
Definition ha_service.h:1270
isc::ha::HAService::server_type_
HAServerType server_type_
DHCP server type.
Definition ha_service.h:1283
isc::ha::HAService::sync_complete_notified_
bool sync_complete_notified_
An indicator that a partner sent ha-sync-complete-notify command.
Definition ha_service.h:1422
isc::ha::HAService::shouldTerminate
bool shouldTerminate() const
Indicates if the server should transition to the terminated state.
Definition ha_service.cc:1164
isc::ha::HAService::processScopes
data::ConstElementPtr processScopes(const std::vector< std::string > &scopes)
Processes ha-scopes command and returns a response.
Definition ha_service.cc:2741
isc::ha::HAService::network_state_
dhcp::NetworkStatePtr network_state_
Pointer to the state of the DHCP service (enabled/disabled).
Definition ha_service.h:1277
isc::ha::HAService::processSynchronize
data::ConstElementPtr processSynchronize(const std::string &server_name, const unsigned int max_period)
Processes ha-sync command and returns a response.
Definition ha_service.cc:2385
isc::ha::HAService::scheduleHeartbeat
void scheduleHeartbeat()
Schedules asynchronous heartbeat to a peer if it is not scheduled.
Definition ha_service.cc:1936
isc::ha::HAService::asyncSyncCompleteNotify
void asyncSyncCompleteNotify(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, PostRequestCallback post_request_action)
Schedules asynchronous "ha-sync-complete-notify" command to the specified server.
Definition ha_service.cc:3081
isc::ha::HAService::query_filter_
QueryFilter query_filter_
Selects queries to be processed/dropped.
Definition ha_service.h:1296
isc::ha::HAService::HA_MAINTENANCE_NOTIFY_EVT
static const int HA_MAINTENANCE_NOTIFY_EVT
ha-maintenance-notify command received.
Definition ha_service.h:68
isc::ha::HAService::HA_SYNCED_PARTNER_UNAVAILABLE_EVT
static const int HA_SYNCED_PARTNER_UNAVAILABLE_EVT
The heartbeat command failed after receiving ha-sync-complete-notify command from the partner.
Definition ha_service.h:78
isc::ha::HAService::processMaintenanceNotify
data::ConstElementPtr processMaintenanceNotify(const bool cancel, const std::string &state)
Processes ha-maintenance-notify command and returns a response.
Definition ha_service.cc:2762
isc::ha::HAService::conditionalLogPausedState
void conditionalLogPausedState() const
Logs if the server is paused in the current state.
Definition ha_service.cc:1037
isc::ha::HAService::unpause
bool unpause()
Unpauses the HA state machine with logging.
Definition ha_service.cc:1026
isc::ha::HAService::HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED
static const int HA_CONTROL_RESULT_MAINTENANCE_NOT_ALLOWED
Control result returned in response to ha-maintenance-notify.
Definition ha_service.h:81
isc::ha::HAService::serveDefaultScopes
void serveDefaultScopes()
Instructs the HA service to serve default scopes.
Definition ha_service.cc:1049
isc::ha::HAService::asyncSendLeaseUpdates
size_t asyncSendLeaseUpdates(const dhcp::Pkt4Ptr &query, const dhcp::Lease4CollectionPtr &leases, const dhcp::Lease4CollectionPtr &deleted_leases, const hooks::ParkingLotHandlePtr &parking_lot)
Schedules asynchronous IPv4 leases updates.
Definition ha_service.cc:1218
isc::ha::HAService::pendingRequestSize
size_t pendingRequestSize()
Get the number of entries in the pending request map.
Definition ha_service.cc:3355
isc::ha::HAService::HA_SYNCING_SUCCEEDED_EVT
static const int HA_SYNCING_SUCCEEDED_EVT
Lease database synchronization succeeded.
Definition ha_service.h:65
isc::ha::HAService::sendHAReset
bool sendHAReset()
Sends ha-reset command to partner synchronously.
Definition ha_service.cc:2718
isc::ha::HAService::PostRequestCallback
std::function< void(const bool, const std::string &, const int)> PostRequestCallback
Callback invoked when request was sent and a response received or an error occurred.
Definition ha_service.h:95
isc::ha::HAService::io_service_
asiolink::IOServicePtr io_service_
Pointer to the IO service object shared between this hooks library and the DHCP server.
Definition ha_service.h:1274
isc::ha::HAService::localDisableDHCPService
void localDisableDHCPService()
Disables local DHCP service.
Definition ha_service.cc:2111
isc::ha::HAService::communication_state_
CommunicationStatePtr communication_state_
Holds communication state with a peer.
Definition ha_service.h:1293
isc::ha::HAService::logFailedLeaseUpdates
void logFailedLeaseUpdates(const dhcp::PktPtr &query, const data::ConstElementPtr &args) const
Log failed lease updates.
Definition ha_service.cc:1637
isc::ha::HAService::lease_update_backlog_
LeaseUpdateBacklog lease_update_backlog_
Backlog of DHCP lease updates.
Definition ha_service.h:1413
isc::ha::HAService::~HAService
virtual ~HAService()
Destructor.
Definition ha_service.cc:147
isc::ha::HAService::HA_SYNCING_FAILED_EVT
static const int HA_SYNCING_FAILED_EVT
Lease database synchronization failed.
Definition ha_service.h:62
isc::ha::HAService::HA_MAINTENANCE_CANCEL_EVT
static const int HA_MAINTENANCE_CANCEL_EVT
ha-maintenance-cancel command received.
Definition ha_service.h:74
isc::ha::HAService::asyncSendLeaseUpdatesFromBacklog
void asyncSendLeaseUpdatesFromBacklog(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, PostRequestCallback post_request_action)
Sends lease updates from backlog to partner asynchronously.
Definition ha_service.cc:2539
isc::ha::HAService::processHeartbeat
data::ConstElementPtr processHeartbeat()
Processes ha-heartbeat command and returns a response.
Definition ha_service.cc:1746
isc::ha::HAService::asyncSyncLeasesInternal
void asyncSyncLeasesInternal(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, const unsigned int max_period, const dhcp::LeasePtr &last_lease, PostSyncCallback post_sync_action, const bool dhcp_disabled)
Implements fetching one page of leases during synchronization.
Definition ha_service.cc:2169
isc::ha::HAService::processHAReset
data::ConstElementPtr processHAReset()
Processes ha-reset command and returns a response.
Definition ha_service.cc:1769
isc::ha::HAService::asyncSendSingleLeaseUpdate
size_t asyncSendSingleLeaseUpdate(const dhcp::Pkt4Ptr &query, const dhcp::Lease4Ptr &lease, const hooks::ParkingLotHandlePtr &parking_lot)
Schedules an asynchronous IPv4 lease update.
Definition ha_service.cc:1300
isc::ha::HAService::asyncSendHeartbeat
void asyncSendHeartbeat()
Starts asynchronous heartbeat to a peer.
Definition ha_service.cc:1779
isc::ha::HAService::isPartnerStateInvalid
bool isPartnerStateInvalid() const
Indicates if the partner's state is invalid.
Definition ha_service.cc:1185
isc::ha::HAService::startClientAndListener
void startClientAndListener()
Start the client and(or) listener instances.
Definition ha_service.cc:3415
isc::ha::HAService::verifyAsyncResponse
data::ConstElementPtr verifyAsyncResponse(const http::HttpResponsePtr &response, int &rcode)
Checks if the response is valid or contains an error.
Definition ha_service.cc:3185
isc::ha::HAService::resumeClientAndListener
void resumeClientAndListener()
Resumes client and(or) listener thread pool operations.
Definition ha_service.cc:3450
isc::ha::HAService::processStatusGet
data::ConstElementPtr processStatusGet() const
Processes status-get command and returns a response.
Definition ha_service.cc:1689
isc::ha::HAService::getPendingRequest
int getPendingRequest(const QueryPtrType &query)
Get the number of scheduled requests for a given query.
Definition ha_service.cc:3366
isc::ha::HAService::lease_sync_filter_
LeaseSyncFilter lease_sync_filter_
Lease synchronization filter used in hub-and-spoke model.
Definition ha_service.h:1299
isc::ha::HAService::synchronize
int synchronize(std::string &status_message, const HAConfig::PeerConfigPtr &remote_config, const unsigned int max_period)
Synchronizes lease database with a partner.
Definition ha_service.cc:2404
isc::ha::HAService::shouldSendLeaseUpdates
bool shouldSendLeaseUpdates(const HAConfig::PeerConfigPtr &peer_config) const
Checks if the lease updates should be sent as result of leases allocation or release.
Definition ha_service.cc:1592
isc::ha::HAService::serveFailoverScopes
void serveFailoverScopes()
Instructs the HA service to serve failover scopes.
Definition ha_service.cc:1054
isc::ha::HAService::localEnableDHCPService
void localEnableDHCPService()
Enables local DHCP service.
Definition ha_service.cc:2116
isc::ha::HAService::HA_LEASE_UPDATES_COMPLETE_EVT
static const int HA_LEASE_UPDATES_COMPLETE_EVT
Finished lease updates commands.
Definition ha_service.h:59
isc::ha::HAService::HAService
HAService(const unsigned int id, const asiolink::IOServicePtr &io_service, const dhcp::NetworkStatePtr &network_state, const HAConfigPtr &config, const HAServerType &server_type=HAServerType::DHCPv4)
Constructor.
Definition ha_service.cc:76
isc::ha::HAService::socketReadyHandler
void socketReadyHandler(int tcp_native_fd)
IfaceMgr external socket ready callback handler.
Definition ha_service.cc:3339
isc::ha::HAService::client_
http::HttpClientPtr client_
HTTP client instance used to send HA commands and lease updates.
Definition ha_service.h:1286
isc::ha::HAService::updatePendingRequest
void updatePendingRequest(QueryPtrType &query)
Update pending request counter for this query.
Definition ha_service.cc:1413
isc::ha::HAService::shouldPartnerDown
bool shouldPartnerDown() const
Indicates if the server should transition to the partner down state.
Definition ha_service.cc:1137
isc::ha::HAService::startHeartbeat
void startHeartbeat()
Unconditionally starts one heartbeat to a peer.
Definition ha_service.cc:1943
isc::ha::HAService::processSyncCompleteNotify
data::ConstElementPtr processSyncCompleteNotify(const unsigned int origin_id)
Process ha-sync-complete-notify command and returns a response.
Definition ha_service.cc:3164
isc::ha::HAService::processContinue
data::ConstElementPtr processContinue()
Processes ha-continue command and returns a response.
Definition ha_service.cc:2754
isc::ha::HAService::asyncDisableDHCPService
void asyncDisableDHCPService(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, const unsigned int max_period, PostRequestCallback post_request_action)
Schedules asynchronous "dhcp-disable" command to the specified server.
Definition ha_service.cc:1952
isc::ha::HAService::pauseClientAndListener
void pauseClientAndListener()
Pauses client and(or) listener thread pool operations.
Definition ha_service.cc:3432
isc::ha::HAService::PostSyncCallback
std::function< void(const bool, const std::string &, const bool)> PostSyncCallback
Callback invoked when lease database synchronization is complete.
Definition ha_service.h:104
isc::ha::HAService::HA_WAITING_TO_TERMINATED_ST_DELAY_MINUTES
static const int HA_WAITING_TO_TERMINATED_ST_DELAY_MINUTES
A delay in minutes to transition from the waiting to terminated state when the partner remains in ter...
Definition ha_service.h:85
isc::ha::HAService::asyncEnableDHCPService
void asyncEnableDHCPService(http::HttpClient &http_client, const HAConfig::PeerConfigPtr &remote_config, PostRequestCallback post_request_action)
Schedules asynchronous "dhcp-enable" command to the specified server.
Definition ha_service.cc:2033
isc::ha::LeaseUpdateBacklog::OpType
OpType
Type of the lease update (operation type).
Definition lease_update_backlog.h:49
isc::ha::LeaseUpdateBacklog::DELETE
@ DELETE
Definition lease_update_backlog.h:51
isc::ha::LeaseUpdateBacklog::ADD
@ ADD
Definition lease_update_backlog.h:50
isc::ha::QueryFilter::inScope
bool inScope(const dhcp::Pkt4Ptr &query4, std::string &scope_class) const
Checks if this server should process the DHCPv4 query.
Definition query_filter.cc:363
isc::http::HostHttpHeader
Represents HTTP Host header.
Definition http_header.h:68
isc::http::HttpClient
HTTP client class.
Definition lib/http/client.h:86
isc::http::HttpClient::stop
void stop()
Halts client-side IO activity.
Definition client.cc:2036
isc::http::HttpClient::asyncSendRequest
void asyncSendRequest(const Url &url, const asiolink::TlsContextPtr &tls_context, const HttpRequestPtr &request, const HttpResponsePtr &response, const RequestHandler &request_callback, const RequestTimeout &request_timeout=RequestTimeout(10000), const ConnectHandler &connect_callback=ConnectHandler(), const HandshakeHandler &handshake_callback=HandshakeHandler(), const CloseHandler &close_callback=CloseHandler())
Queues new asynchronous HTTP request for a given URL.
Definition client.cc:1975
isc::http::HttpDateTime
This class parses and generates time values used in HTTP.
Definition date_time.h:41
isc::http::HttpDateTime::rfc1123Format
std::string rfc1123Format() const
Returns time value formatted as specified in RFC 1123.
Definition date_time.cc:30
isc::http::HttpRequest::Method::HTTP_POST
@ HTTP_POST
Definition request.h:63
isc::util::MultiThreadingMgr::instance
static MultiThreadingMgr & instance()
Returns a single instance of Multi Threading Manager.
Definition multi_threading_mgr.cc:25
isc::util::MultiThreadingMgr::removeCriticalSectionCallbacks
void removeCriticalSectionCallbacks(const std::string &name)
Removes the set of callbacks associated with a given name from the list of CriticalSection callbacks.
Definition multi_threading_mgr.cc:224
isc::util::MultiThreadingMgr::addCriticalSectionCallbacks
void addCriticalSectionCallbacks(const std::string &name, const CSCallbackSet::Callback &check_cb, const CSCallbackSet::Callback &entry_cb, const CSCallbackSet::Callback &exit_cb)
Adds a set of callbacks to the list of CriticalSection callbacks.
Definition multi_threading_mgr.cc:216
isc::util::StateModel::getEvent
const EventPtr & getEvent(unsigned int value)
Fetches the event referred to by value.
Definition state_model.cc:186
isc::util::StateModel::getStateLabel
std::string getStateLabel(const int state) const
Fetches the label associated with an state value.
Definition state_model.cc:421
isc::util::StateModel::unpauseModel
void unpauseModel()
Unpauses state model.
Definition state_model.cc:276
isc::util::StateModel::runModel
virtual void runModel(unsigned int event)
Processes events through the state model.
Definition state_model.cc:112
isc::util::StateModel::isModelPaused
bool isModelPaused() const
Returns whether or not the model is paused.
Definition state_model.cc:415
isc::util::StateModel::defineEvents
virtual void defineEvents()
Populates the set of events.
Definition state_model.cc:229
isc::util::StateModel::postNextEvent
void postNextEvent(unsigned int event)
Sets the next event to the given event value.
Definition state_model.cc:320
isc::util::StateModel::defineState
void defineState(unsigned int value, const std::string &label, StateHandler handler, const StatePausing &state_pausing=STATE_PAUSE_NEVER)
Adds an state value and associated label to the set of states.
Definition state_model.cc:196
isc::util::StateModel::getState
const StatePtr getState(unsigned int value)
Fetches the state referred to by value.
Definition state_model.cc:213
isc::util::StateModel::doOnExit
bool doOnExit()
Checks if on exit flag is true.
Definition state_model.cc:347
isc::util::StateModel::getNextEvent
unsigned int getNextEvent() const
Fetches the model's next event.
Definition state_model.cc:373
isc::util::StateModel::defineEvent
void defineEvent(unsigned int value, const std::string &label)
Adds an event value and associated label to the set of events.
Definition state_model.cc:170
isc::util::StateModel::transition
void transition(unsigned int state, unsigned int event)
Sets up the model to transition into given state with a given event.
Definition state_model.cc:264
isc::util::StateModel::verifyEvents
virtual void verifyEvents()
Validates the contents of the set of events.
Definition state_model.cc:237
isc::util::StateModel::doOnEntry
bool doOnEntry()
Checks if on entry flag is true.
Definition state_model.cc:339
isc::util::StateModel::NOP_EVT
static const int NOP_EVT
Signifies that no event has occurred.
Definition state_model.h:292
isc::util::StateModel::startModel
void startModel(const int start_state)
Begins execution of the model.
Definition state_model.cc:100
isc::util::StateModel::defineStates
virtual void defineStates()
Populates the set of states.
Definition state_model.cc:245
isc::util::StateModel::getLastEvent
unsigned int getLastEvent() const
Fetches the model's last event.
Definition state_model.cc:367
isc::util::StateModel::getPrevState
unsigned int getPrevState() const
Fetches the model's previous state.
Definition state_model.cc:361
isc::util::StateModel::getCurrState
unsigned int getCurrState() const
Fetches the model's current state.
Definition state_model.cc:355
isc::util::Stopwatch
Utility class to measure code execution times.
Definition stopwatch.h:35
isc::util::Stopwatch::stop
void stop()
Stops the stopwatch.
Definition stopwatch.cc:34
isc::util::Stopwatch::logFormatLastDuration
std::string logFormatLastDuration() const
Returns the last measured duration in the format directly usable in log messages.
Definition stopwatch.cc:74
cmd_response_creator.h
command_creator.h
command_interpreter.h
This file contains several functions and constants that are used for handling commands and responses ...
data.h
date_time.h
exceptions.h
isc_throw
#define isc_throw(type, stream)
A shortcut macro to insert known values into exception arguments.
Definition exceptions/exceptions.h:210
ha_log.h
ha_service.h
ha_service_states.h
iface_mgr.h
lease_mgr.h
An abstract API for lease database.
lease_mgr_factory.h
LOG_ERROR
#define LOG_ERROR(LOGGER, MESSAGE)
Macro to conveniently test error output and log it.
Definition macros.h:32
LOG_INFO
#define LOG_INFO(LOGGER, MESSAGE)
Macro to conveniently test info output and log it.
Definition macros.h:20
LOG_WARN
#define LOG_WARN(LOGGER, MESSAGE)
Macro to conveniently test warn output and log it.
Definition macros.h:26
multi_threading_mgr.h
isc::asiolink
Definition addr_utilities.cc:170
isc::asiolink::IOServicePtr
boost::shared_ptr< IOService > IOServicePtr
Defines a smart pointer to an IOService instance.
Definition io_service.h:28
isc::config
Definition command_interpreter.cc:23
isc::config::CONTROL_RESULT_EMPTY
const int CONTROL_RESULT_EMPTY
Status code indicating that the specified command was completed correctly, but failed to produce any ...
Definition command_interpreter.h:53
isc::config::CONTROL_TEXT
const char * CONTROL_TEXT
String used for storing textual description ("text").
Definition command_interpreter.cc:27
isc::config::parseAnswer
ConstElementPtr parseAnswer(int &rcode, const ConstElementPtr &msg)
Parses a standard config/command level answer and returns arguments or text status code.
Definition command_interpreter.cc:96
isc::config::TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST
constexpr long TIMEOUT_DEFAULT_HTTP_CLIENT_REQUEST
Timeout for the HTTP clients awaiting a response to a request.
Definition timeouts.h:38
isc::config::CONTROL_RESULT_ERROR
const int CONTROL_RESULT_ERROR
Status code indicating a general failure.
Definition command_interpreter.h:45
isc::config::createAnswer
ConstElementPtr createAnswer()
Creates a standard config/command level success answer message (i.e.
Definition command_interpreter.cc:54
isc::config::CONTROL_RESULT_CONFLICT
const int CONTROL_RESULT_CONFLICT
Status code indicating that the command was unsuccessful due to a conflict between the command argume...
Definition command_interpreter.h:59
isc::config::CONTROL_RESULT_COMMAND_UNSUPPORTED
const int CONTROL_RESULT_COMMAND_UNSUPPORTED
Status code indicating that the specified command is not supported.
Definition command_interpreter.h:48
isc::config::CONTROL_RESULT
const char * CONTROL_RESULT
String used for result, i.e. integer status ("result").
Definition command_interpreter.cc:26
isc::config::CONTROL_RESULT_SUCCESS
const int CONTROL_RESULT_SUCCESS
Status code indicating a successful operation.
Definition command_interpreter.h:42
isc::data
Definition base_stamped_element.cc:12
isc::data::ConstElementPtr
boost::shared_ptr< const Element > ConstElementPtr
Definition data.h:29
isc::data::ElementPtr
boost::shared_ptr< Element > ElementPtr
Definition data.h:28
isc::dhcp
Definition dhcp4/client_handler.cc:20
isc::dhcp::PktPtr
boost::shared_ptr< isc::dhcp::Pkt > PktPtr
A pointer to either Pkt4 or Pkt6 packet.
Definition pkt.h:998
isc::dhcp::ClientClass
std::string ClientClass
Defines a single class name.
Definition classify.h:43
isc::dhcp::Lease4CollectionPtr
boost::shared_ptr< Lease4Collection > Lease4CollectionPtr
A shared pointer to the collection of IPv4 leases.
Definition lease.h:523
isc::dhcp::Pkt4Ptr
boost::shared_ptr< Pkt4 > Pkt4Ptr
A pointer to Pkt4 object.
Definition pkt4.h:555
isc::dhcp::Lease6Ptr
boost::shared_ptr< Lease6 > Lease6Ptr
Pointer to a Lease6 structure.
Definition lease.h:528
isc::dhcp::LeasePtr
boost::shared_ptr< Lease > LeasePtr
Pointer to the lease object.
Definition lease.h:25
isc::dhcp::NetworkStatePtr
boost::shared_ptr< NetworkState > NetworkStatePtr
Pointer to the NetworkState object.
Definition network_state.h:220
isc::dhcp::Lease6CollectionPtr
boost::shared_ptr< Lease6Collection > Lease6CollectionPtr
A shared pointer to the collection of IPv6 leases.
Definition lease.h:696
isc::dhcp::Pkt6Ptr
boost::shared_ptr< Pkt6 > Pkt6Ptr
A pointer to Pkt6 packet.
Definition pkt6.h:31
isc::dhcp::Lease4Collection
std::vector< Lease4Ptr > Lease4Collection
A collection of IPv4 leases.
Definition lease.h:520
isc::dhcp::Lease4Ptr
boost::shared_ptr< Lease4 > Lease4Ptr
Pointer to a Lease4 structure.
Definition lease.h:315
isc::ha
Definition command_creator.cc:20
isc::ha::HA_INVALID_PARTNER_STATE_LOAD_BALANCING
const isc::log::MessageID HA_INVALID_PARTNER_STATE_LOAD_BALANCING
Definition ha_messages.h:52
isc::ha::HA_RESUME_CLIENT_LISTENER_FAILED
const isc::log::MessageID HA_RESUME_CLIENT_LISTENER_FAILED
Definition ha_messages.h:108
isc::ha::HA_LOCAL_DHCP_ENABLE
const isc::log::MessageID HA_LOCAL_DHCP_ENABLE
Definition ha_messages.h:91
isc::ha::HA_LEASES_BACKLOG_NOTHING_TO_SEND
const isc::log::MessageID HA_LEASES_BACKLOG_NOTHING_TO_SEND
Definition ha_messages.h:68
isc::ha::HA_LEASES_BACKLOG_FAILED
const isc::log::MessageID HA_LEASES_BACKLOG_FAILED
Definition ha_messages.h:67
isc::ha::HA_SYNC_FAILED
const isc::log::MessageID HA_SYNC_FAILED
Definition ha_messages.h:130
isc::ha::HA_TERMINATED_RESTART_PARTNER
const isc::log::MessageID HA_TERMINATED_RESTART_PARTNER
Definition ha_messages.h:136
isc::ha::HA_PASSIVE_BACKUP_ST
const int HA_PASSIVE_BACKUP_ST
In passive-backup state with a single active server and backup servers.
Definition ha_service_states.h:38
isc::ha::HA_HOT_STANDBY_ST
const int HA_HOT_STANDBY_ST
Hot standby state.
Definition ha_service_states.h:23
isc::ha::HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY
const isc::log::MessageID HA_INVALID_PARTNER_STATE_COMMUNICATION_RECOVERY
Definition ha_messages.h:50
isc::ha::HA_LEASES_BACKLOG_SUCCESS
const isc::log::MessageID HA_LEASES_BACKLOG_SUCCESS
Definition ha_messages.h:70
isc::ha::HA_COMMUNICATION_RECOVERY_ST
const int HA_COMMUNICATION_RECOVERY_ST
Communication recovery state.
Definition ha_service_states.h:20
isc::ha::HA_STATE_MACHINE_CONTINUED
const isc::log::MessageID HA_STATE_MACHINE_CONTINUED
Definition ha_messages.h:111
isc::ha::ha_logger
isc::log::Logger ha_logger("ha-hooks")
Definition ha_log.h:17
isc::ha::HA_LEASES_SYNC_FAILED
const isc::log::MessageID HA_LEASES_SYNC_FAILED
Definition ha_messages.h:73
isc::ha::HA_SYNC_SUCCESSFUL
const isc::log::MessageID HA_SYNC_SUCCESSFUL
Definition ha_messages.h:133
isc::ha::HA_UNAVAILABLE_ST
const int HA_UNAVAILABLE_ST
Special state indicating that this server is unable to communicate with the partner.
Definition ha_service_states.h:54
isc::ha::HA_CONFIG_LEASE_UPDATES_DISABLED_REMINDER
const isc::log::MessageID HA_CONFIG_LEASE_UPDATES_DISABLED_REMINDER
Definition ha_messages.h:34
isc::ha::HA_SERVICE_STARTED
const isc::log::MessageID HA_SERVICE_STARTED
Definition ha_messages.h:110
isc::ha::HA_TERMINATED_ST
const int HA_TERMINATED_ST
HA service terminated state.
Definition ha_service_states.h:47
isc::ha::HA_IN_MAINTENANCE_ST
const int HA_IN_MAINTENANCE_ST
In maintenance state.
Definition ha_service_states.h:29
isc::ha::HA_LOAD_BALANCING_ST
const int HA_LOAD_BALANCING_ST
Load balancing state.
Definition ha_service_states.h:26
isc::ha::HA_DHCP_ENABLE_FAILED
const isc::log::MessageID HA_DHCP_ENABLE_FAILED
Definition ha_messages.h:43
isc::ha::HA_LEASE_UPDATE_DELETE_FAILED_ON_PEER
const isc::log::MessageID HA_LEASE_UPDATE_DELETE_FAILED_ON_PEER
Definition ha_messages.h:83
isc::ha::HA_LEASES_BACKLOG_START
const isc::log::MessageID HA_LEASES_BACKLOG_START
Definition ha_messages.h:69
isc::ha::HA_SYNC_START
const isc::log::MessageID HA_SYNC_START
Definition ha_messages.h:132
isc::ha::HA_HEARTBEAT_FAILED
const isc::log::MessageID HA_HEARTBEAT_FAILED
Definition ha_messages.h:45
isc::ha::HA_PARTNER_DOWN_ST
const int HA_PARTNER_DOWN_ST
Partner down state.
Definition ha_service_states.h:32
isc::ha::HA_LEASE_UPDATES_ENABLED
const isc::log::MessageID HA_LEASE_UPDATES_ENABLED
Definition ha_messages.h:79
isc::ha::HA_INVALID_PARTNER_STATE_HOT_STANDBY
const isc::log::MessageID HA_INVALID_PARTNER_STATE_HOT_STANDBY
Definition ha_messages.h:51
isc::ha::HA_STATE_MACHINE_PAUSED
const isc::log::MessageID HA_STATE_MACHINE_PAUSED
Definition ha_messages.h:112
isc::ha::HA_TERMINATED
const isc::log::MessageID HA_TERMINATED
Definition ha_messages.h:134
isc::ha::HA_DHCP_DISABLE_FAILED
const isc::log::MessageID HA_DHCP_DISABLE_FAILED
Definition ha_messages.h:41
isc::ha::HAConfigPtr
boost::shared_ptr< HAConfig > HAConfigPtr
Pointer to the High Availability configuration structure.
Definition ha_config.h:37
isc::ha::HA_MAINTENANCE_STARTED_IN_PARTNER_DOWN
const isc::log::MessageID HA_MAINTENANCE_STARTED_IN_PARTNER_DOWN
Definition ha_messages.h:100
isc::ha::HA_PARTNER_IN_MAINTENANCE_ST
const int HA_PARTNER_IN_MAINTENANCE_ST
Partner in-maintenance state.
Definition ha_service_states.h:35
isc::ha::HA_MAINTENANCE_NOTIFY_FAILED
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_FAILED
Definition ha_messages.h:96
isc::ha::HA_WAITING_ST
const int HA_WAITING_ST
Server waiting state, i.e. waiting for another server to be ready.
Definition ha_service_states.h:50
isc::ha::HAServerType
HAServerType
Lists possible server types for which HA service is created.
Definition ha_server_type.h:14
isc::ha::HAServerType::DHCPv4
@ DHCPv4
Definition ha_server_type.h:15
isc::ha::HA_BACKUP_ST
const int HA_BACKUP_ST
Backup state.
Definition ha_service_states.h:17
isc::ha::HA_PAUSE_CLIENT_LISTENER_ILLEGAL
const isc::log::MessageID HA_PAUSE_CLIENT_LISTENER_ILLEGAL
Definition ha_messages.h:104
isc::ha::HA_PAUSE_CLIENT_LISTENER_FAILED
const isc::log::MessageID HA_PAUSE_CLIENT_LISTENER_FAILED
Definition ha_messages.h:103
isc::ha::HA_MAINTENANCE_SHUTDOWN_SAFE
const isc::log::MessageID HA_MAINTENANCE_SHUTDOWN_SAFE
Definition ha_messages.h:98
isc::ha::HA_MAINTENANCE_NOTIFY_CANCEL_FAILED
const isc::log::MessageID HA_MAINTENANCE_NOTIFY_CANCEL_FAILED
Definition ha_messages.h:94
isc::ha::HA_LEASE_UPDATE_CONFLICT
const isc::log::MessageID HA_LEASE_UPDATE_CONFLICT
Definition ha_messages.h:81
isc::ha::HA_LEASE_UPDATES_DISABLED
const isc::log::MessageID HA_LEASE_UPDATES_DISABLED
Definition ha_messages.h:78
isc::ha::HA_LOCAL_DHCP_DISABLE
const isc::log::MessageID HA_LOCAL_DHCP_DISABLE
Definition ha_messages.h:90
isc::ha::HA_SYNCING_ST
const int HA_SYNCING_ST
Synchronizing database state.
Definition ha_service_states.h:44
isc::ha::HA_RESET_FAILED
const isc::log::MessageID HA_RESET_FAILED
Definition ha_messages.h:106
isc::ha::HA_STATE_TRANSITION
const isc::log::MessageID HA_STATE_TRANSITION
Definition ha_messages.h:113
isc::ha::HA_CONFIG_LEASE_SYNCING_DISABLED_REMINDER
const isc::log::MessageID HA_CONFIG_LEASE_SYNCING_DISABLED_REMINDER
Definition ha_messages.h:31
isc::ha::stateToString
std::string stateToString(int state)
Returns state name.
Definition ha_service_states.cc:14
isc::ha::HA_READY_ST
const int HA_READY_ST
Server ready state, i.e. synchronized database, can enable DHCP service.
Definition ha_service_states.h:41
isc::ha::HA_TERMINATED_PARTNER_DID_NOT_RESTART
const isc::log::MessageID HA_TERMINATED_PARTNER_DID_NOT_RESTART
Definition ha_messages.h:135
isc::ha::HA_SYNC_COMPLETE_NOTIFY_FAILED
const isc::log::MessageID HA_SYNC_COMPLETE_NOTIFY_FAILED
Definition ha_messages.h:128
isc::ha::HA_MAINTENANCE_STARTED
const isc::log::MessageID HA_MAINTENANCE_STARTED
Definition ha_messages.h:99
isc::ha::HA_LEASE_UPDATE_CREATE_UPDATE_FAILED_ON_PEER
const isc::log::MessageID HA_LEASE_UPDATE_CREATE_UPDATE_FAILED_ON_PEER
Definition ha_messages.h:82
isc::ha::HA_LEASE_UPDATE_FAILED
const isc::log::MessageID HA_LEASE_UPDATE_FAILED
Definition ha_messages.h:84
isc::ha::HA_STATE_TRANSITION_PASSIVE_BACKUP
const isc::log::MessageID HA_STATE_TRANSITION_PASSIVE_BACKUP
Definition ha_messages.h:114
isc::hooks
Definition callout_handle.cc:21
isc::hooks::ParkingLotHandlePtr
boost::shared_ptr< ParkingLotHandle > ParkingLotHandlePtr
Pointer to the parking lot handle.
Definition parking_lots.h:381
isc::http
Definition auth_config.h:19
isc::http::PostHttpRequestJsonPtr
boost::shared_ptr< PostHttpRequestJson > PostHttpRequestJsonPtr
Pointer to PostHttpRequestJson.
Definition post_request_json.h:29
isc::http::HttpResponseJsonPtr
boost::shared_ptr< HttpResponseJson > HttpResponseJsonPtr
Pointer to the HttpResponseJson object.
Definition response_json.h:27
isc::http::HttpResponsePtr
boost::shared_ptr< HttpResponse > HttpResponsePtr
Pointer to the HttpResponse object.
Definition response.h:81
isc::log
Definition buffer_appender_impl.cc:17
isc::log::MessageID
const char * MessageID
Definition message_types.h:15
isc::util
Definition time_utils.cc:43
isc::util::ptimeToText
std::string ptimeToText(boost::posix_time::ptime t, size_t fsecs_precision=MAX_FSECS_PRECISION)
Converts ptime structure to text.
Definition boost_time_utils.cc:14
isc
Defines the logger used by the top-level component of kea-lfc.
Definition agent_parser.cc:148
post_request_json.h
response_json.h
stopwatch.h
isc::dhcp::Lease::STATE_RELEASED
static const uint32_t STATE_RELEASED
Released lease held in the database for lease affinity.
Definition lease.h:78
isc::http::HttpClient::RequestTimeout
HTTP request/response timeout value.
Definition lib/http/client.h:89
isc::http::HttpVersion::HTTP_11
static const HttpVersion & HTTP_11()
HTTP version 1.1.
Definition http_types.h:59
timeouts.h