Map country code to different Twilio resources (#1976)

Many countries are introducing different requirements for SMS senders to
register and/or use alpha numeric ids, short codes or regional numbers
or face being blocked. The changes in this PR will give us more
flexibility by allowing us to map to different resources in Twilio based
on the phone number we are trying to reach. For this first
implementation the selection is made based on country code of the
recipient. Verification and phone calls were given the same treatment
although the immediate need is for SMS. Senders with no country code set
can be used as catch-all defaults. This also falls back to the
configured live settings/environment variables if not configured.

Possible future additions:
- Move through list of trying multiple senders before failing
notification
- Easily expanded to allow per-organization or per-user resources to let
users and tenants configure their own Twilio
- Add UI + replace live settings so users can configure their own
settings
- More selection criteria if needed

TODO:

- [x] Add+Fix Tests
- [x] Verify changes are compatible with #1713

Author: mderynck

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Add models and framework to use different services (Phone, SMS, Verify) in Twilio depending on
+the destination country code by @mderynck ([#1976](https://github.com/grafana/oncall/pull/1976))
 - Prometheus exporter backend for alert groups related metrics
 
 ### Fixed

engine/apps/base/tests/test_live_settings.py

@@ -67,6 +67,6 @@ def test_twilio_respects_changed_credentials(settings):
     live_settings.TWILIO_AUTH_TOKEN = "new_twilio_auth_token"
     live_settings.TWILIO_NUMBER = "new_twilio_number"
 
-    assert twilio_client._twilio_api_client.username == "new_twilio_account_sid"
-    assert twilio_client._twilio_api_client.password == "new_twilio_auth_token"
-    assert twilio_client._twilio_number == "new_twilio_number"
+    assert twilio_client._default_twilio_api_client.username == "new_twilio_account_sid"
+    assert twilio_client._default_twilio_api_client.password == "new_twilio_auth_token"
+    assert twilio_client._default_twilio_number == "new_twilio_number"

engine/apps/twilioapp/migrations/0005_twilioaccount_twiliophonecallsender_twiliosmssender_twilioverificationsender.py

@@ -0,0 +1,64 @@
+# Generated by Django 3.2.19 on 2023-05-25 15:32
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('twilioapp', '0004_twiliophonecall_twiliosms'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TwilioAccount',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=100)),
+                ('account_sid', models.CharField(max_length=64, unique=True)),
+                ('auth_token', models.CharField(default=None, max_length=64, null=True)),
+                ('api_key_sid', models.CharField(default=None, max_length=64, null=True)),
+                ('api_key_secret', models.CharField(default=None, max_length=64, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='TwilioVerificationSender',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(default='Default', max_length=100)),
+                ('country_code', models.CharField(default=None, max_length=16, null=True)),
+                ('verify_service_sid', models.CharField(max_length=64)),
+                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='twilioapp_twilioverificationsender_account', to='twilioapp.twilioaccount')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='TwilioSmsSender',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(default='Default', max_length=100)),
+                ('country_code', models.CharField(default=None, max_length=16, null=True)),
+                ('sender', models.CharField(max_length=16)),
+                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='twilioapp_twiliosmssender_account', to='twilioapp.twilioaccount')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='TwilioPhoneCallSender',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(default='Default', max_length=100)),
+                ('country_code', models.CharField(default=None, max_length=16, null=True)),
+                ('number', models.CharField(max_length=16)),
+                ('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='twilioapp_twiliophonecallsender_account', to='twilioapp.twilioaccount')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

engine/apps/twilioapp/models/__init__.py

@@ -1,3 +1,10 @@
 from .twilio_log_record import TwilioLogRecord  # noqa: F401
 from .twilio_phone_call import TwilioCallStatuses, TwilioPhoneCall  # noqa: F401
+from .twilio_sender import (  # noqa: F401
+    TwilioAccount,
+    TwilioPhoneCallSender,
+    TwilioSender,
+    TwilioSmsSender,
+    TwilioVerificationSender,
+)
 from .twilio_sms import TwilioSMS, TwilioSMSstatuses  # noqa: F401

engine/apps/twilioapp/models/twilio_sender.py

@@ -0,0 +1,41 @@
+from django.db import models
+from twilio.rest import Client
+
+
+class TwilioAccount(models.Model):
+    name = models.CharField(max_length=100)
+    account_sid = models.CharField(max_length=64, null=False, blank=False, unique=True)
+    auth_token = models.CharField(max_length=64, null=True, default=None)
+    api_key_sid = models.CharField(max_length=64, null=True, default=None)
+    api_key_secret = models.CharField(max_length=64, null=True, default=None)
+
+    def get_twilio_api_client(self):
+        if self.api_key_sid and self.api_key_secret:
+            return Client(self.api_key_sid, self.api_key_secret, self.account_sid)
+        else:
+            return Client(self.account_sid, self.auth_token)
+
+
+class TwilioSender(models.Model):
+    name = models.CharField(max_length=100, null=False, default="Default")
+    # Note: country_code does not have + prefix here
+    country_code = models.CharField(max_length=16, null=True, default=None)
+    account = models.ForeignKey(
+        "twilioapp.TwilioAccount", on_delete=models.CASCADE, related_name="%(app_label)s_%(class)s_account"
+    )
+
+    class Meta:
+        abstract = True
+
+
+class TwilioSmsSender(TwilioSender):
+    # Sender for sms is phone number, short code or alphanumeric id
+    sender = models.CharField(max_length=16, null=False, blank=False)
+
+
+class TwilioPhoneCallSender(TwilioSender):
+    number = models.CharField(max_length=16, null=False, blank=False)
+
+
+class TwilioVerificationSender(TwilioSender):
+    verify_service_sid = models.CharField(max_length=64, null=False, blank=False)

engine/apps/twilioapp/phone_provider.py

@@ -2,6 +2,8 @@
 import urllib.parse
 from string import digits
 
+from django.apps import apps
+from django.db.models import F, Q
 from phonenumbers import COUNTRY_CODE_TO_REGION_CODE
 from twilio.base.exceptions import TwilioRestException
 from twilio.rest import Client
@@ -95,9 +97,10 @@ def finish_verification(self, number: str, code: str):
         normalized_number, _ = self._normalize_phone_number(number)
         if normalized_number:
             try:
-                verification_check = self._twilio_api_client.verify.services(
-                    live_settings.TWILIO_VERIFY_SERVICE_SID
-                ).verification_checks.create(to=normalized_number, code=code)
+                client, verify_service_sid = self._verify_sender(number)
+                verification_check = client.verify.services(verify_service_sid).verification_checks.create(
+                    to=normalized_number, code=code
+                )
                 logger.info(f"TwilioPhoneProvider.finish_verification: verification_status {verification_check.status}")
                 if verification_check.status == "approved":
                     return normalized_number
@@ -133,46 +136,45 @@ def _message_to_twiml(self, message: str, with_gather=False):
         )
 
     def _call_create(self, twiml_query: str, to: str, with_callback: bool):
+        client, from_ = self._phone_sender(to)
         url = "http://twimlets.com/echo?Twiml=" + twiml_query
         if with_callback:
             status_callback = get_call_status_callback_url()
             status_callback_events = ["initiated", "ringing", "answered", "completed"]
-            return self._twilio_api_client.calls.create(
+            return client.calls.create(
                 url=url,
                 to=to,
-                from_=self._twilio_number,
+                from_=from_,
                 method="GET",
                 status_callback=status_callback,
                 status_callback_event=status_callback_events,
                 status_callback_method="POST",
             )
         else:
-            return self._twilio_api_client.calls.create(
+            return client.calls.create(
                 url=url,
                 to=to,
-                from_=self._twilio_number,
+                from_=from_,
                 method="GET",
             )
 
     def _messages_create(self, number: str, text: str, with_callback: bool):
+        client, from_ = self._sms_sender(number)
         if with_callback:
             status_callback = get_sms_status_callback_url()
-            return self._twilio_api_client.messages.create(
-                body=text, to=number, from_=self._twilio_number, status_callback=status_callback
-            )
+            return client.messages.create(body=text, to=number, from_=from_, status_callback=status_callback)
         else:
-            return self._twilio_api_client.messages.create(
+            return client.messages.create(
                 body=text,
                 to=number,
-                from_=self._twilio_number,
+                from_=from_,
             )
 
     def _send_verification_code(self, number: str, via: str):
         # https://www.twilio.com/docs/verify/api/verification?code-sample=code-start-a-verification-with-sms&code-language=Python&code-sdk-version=6.x
         try:
-            verification = self._twilio_api_client.verify.services(
-                live_settings.TWILIO_VERIFY_SERVICE_SID
-            ).verifications.create(to=number, channel=via)
+            client, verify_service_sid = self._verify_sender(number)
+            verification = client.verify.services(verify_service_sid).verifications.create(to=number, channel=via)
             logger.info(f"TwilioPhoneProvider._send_verification_code: verification status {verification.status}")
         except TwilioRestException as e:
             logger.error(f"Twilio verification start error: {e} to number {number}")
@@ -202,7 +204,7 @@ def _normalize_phone_number(self, number: str):
     # Use responsibly
     def _parse_number(self, number: str):
         try:
-            response = self._twilio_api_client.lookups.phone_numbers(number).fetch()
+            response = self._default_twilio_api_client.lookups.phone_numbers(number).fetch()
             return True, response.phone_number, self._get_calling_code(response.country_code)
         except TwilioRestException as e:
             if e.code == 20404:
@@ -217,24 +219,50 @@ def _parse_number(self, number: str):
             return False, None, None
 
     @property
-    def _twilio_api_client(self):
+    def _default_twilio_api_client(self):
         if live_settings.TWILIO_API_KEY_SID and live_settings.TWILIO_API_KEY_SECRET:
             return Client(
                 live_settings.TWILIO_API_KEY_SID, live_settings.TWILIO_API_KEY_SECRET, live_settings.TWILIO_ACCOUNT_SID
             )
         else:
             return Client(live_settings.TWILIO_ACCOUNT_SID, live_settings.TWILIO_AUTH_TOKEN)
 
+    @property
+    def _default_twilio_number(self):
+        return live_settings.TWILIO_NUMBER
+
+    def _twilio_sender(self, sender_type, to):
+        _, _, country_code = self._parse_number(to)
+        TwilioSender = apps.get_model("twilioapp", sender_type)
+        sender = (
+            TwilioSender.objects.filter(Q(country_code=country_code) | Q(country_code__isnull=True))
+            .order_by(F("country_code").desc(nulls_last=True))
+            .first()
+        )
+
+        if sender:
+            return sender.account.get_twilio_api_client(), sender
+
+        return self._default_twilio_api_client, None
+
+    def _sms_sender(self, to):
+        client, sender = self._twilio_sender("TwilioSmsSender", to)
+        return client, sender.sender if sender else self._default_twilio_number
+
+    def _phone_sender(self, to):
+        client, sender = self._twilio_sender("TwilioPhoneCallSender", to)
+        return client, sender.number if sender else self._default_twilio_number
+
+    def _verify_sender(self, to):
+        client, sender = self._twilio_sender("TwilioVerificationSender", to)
+        return client, sender.verify_service_sid if sender else live_settings.TWILIO_VERIFY_SERVICE_SID
+
     def _get_calling_code(self, iso):
         for code, isos in COUNTRY_CODE_TO_REGION_CODE.items():
             if iso.upper() in isos:
                 return code
         return None
 
-    @property
-    def _twilio_number(self):
-        return live_settings.TWILIO_NUMBER
-
     def _escape_call_message(self, message):
         # https://www.twilio.com/docs/api/errors/12100
         message = message.replace("&", "&")

engine/apps/twilioapp/tests/conftest.py

@@ -0,0 +1,40 @@
+import pytest
+
+from apps.twilioapp.tests.factories import (
+    TwilioAccountFactory,
+    TwilioPhoneCallSenderFactory,
+    TwilioSmsSenderFactory,
+    TwilioVerificationSenderFactory,
+)
+
+
+@pytest.fixture
+def make_twilio_account():
+    def _make_twilio_account(**kwargs):
+        return TwilioAccountFactory(**kwargs)
+
+    return _make_twilio_account
+
+
+@pytest.fixture
+def make_twilio_phone_call_sender():
+    def _make_twilio_phone_call_sender(**kwargs):
+        return TwilioPhoneCallSenderFactory(**kwargs)
+
+    return _make_twilio_phone_call_sender
+
+
+@pytest.fixture
+def make_twilio_sms_sender():
+    def _make_twilio_sms_sender(**kwargs):
+        return TwilioSmsSenderFactory(**kwargs)
+
+    return _make_twilio_sms_sender
+
+
+@pytest.fixture
+def make_twilio_verification_sender():
+    def _make_twilio_verification_sender(**kwargs):
+        return TwilioVerificationSenderFactory(**kwargs)
+
+    return _make_twilio_verification_sender

engine/apps/twilioapp/tests/factories.py

@@ -0,0 +1,28 @@
+import factory
+
+from apps.twilioapp.models.twilio_sender import (
+    TwilioAccount,
+    TwilioPhoneCallSender,
+    TwilioSmsSender,
+    TwilioVerificationSender,
+)
+
+
+class TwilioAccountFactory(factory.DjangoModelFactory):
+    class Meta:
+        model = TwilioAccount
+
+
+class TwilioPhoneCallSenderFactory(factory.DjangoModelFactory):
+    class Meta:
+        model = TwilioPhoneCallSender
+
+
+class TwilioSmsSenderFactory(factory.DjangoModelFactory):
+    class Meta:
+        model = TwilioSmsSender
+
+
+class TwilioVerificationSenderFactory(factory.DjangoModelFactory):
+    class Meta:
+        model = TwilioVerificationSender